From 17f78c2f778f096353de145b69a3105227061b56 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Mon, 19 Jul 2021 00:26:30 +0800 Subject: [PATCH 001/856] Fix PostgresProvider that use wrong database name --- src/sqlancer/postgres/PostgresProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 683ba95d1..c088fa4b7 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -257,7 +257,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE s.execute(createDatabaseCommand); } con.close(); - int databaseIndex = entryURL.indexOf(entryPath) + 1; + int databaseIndex = entryURL.indexOf(entryDatabaseName); String preDatabaseName = entryURL.substring(0, databaseIndex); String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); testURL = preDatabaseName + databaseName + postDatabaseName; From 748ca241024609ac4d6d7ff5d9402af9dca29903 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Mon, 19 Jul 2021 16:01:47 +0800 Subject: [PATCH 002/856] Fix compilation failure for some generic parameter types in IDEA --- src/sqlancer/arangodb/ArangoDBOptions.java | 5 +++-- src/sqlancer/clickhouse/ClickHouseOptions.java | 10 +++++----- src/sqlancer/mongodb/MongoDBOptions.java | 7 ++++--- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index ba06ceac3..f1759bccb 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -11,6 +11,7 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.arangodb.test.ArangoDBQueryPartitioningWhereTester; +import sqlancer.arangodb.ArangoDBProvider.ArangoDBGlobalState; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; @@ -34,10 +35,10 @@ public List getTestOracleFactory() { return oracles; } - public enum ArangoDBOracleFactory implements OracleFactory { + public enum ArangoDBOracleFactory implements OracleFactory { QUERY_PARTITIONING { @Override - public TestOracle create(ArangoDBProvider.ArangoDBGlobalState globalState) throws Exception { + public TestOracle create(ArangoDBGlobalState globalState) throws Exception { List oracles = new ArrayList<>(); oracles.add(new ArangoDBQueryPartitioningWhereTester(globalState)); return new CompositeTestOracle(oracles, globalState); diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 5c26cb03b..c25905648 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -33,31 +33,31 @@ public class ClickHouseOptions implements DBMSSpecificOptions { TLPWhere { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPWhereOracle(globalState); } }, TLPDistinct { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPDistinctOracle(globalState); } }, TLPGroupBy { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPGroupByOracle(globalState); } }, TLPAggregate { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPAggregateOracle(globalState); } }, TLPHaving { @Override - public TestOracle create(ClickHouseProvider.ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPHavingOracle(globalState); } }; diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 77c565a11..5c618f5f7 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -13,6 +13,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; import sqlancer.mongodb.test.MongoDBDocumentRemovalTester; import sqlancer.mongodb.test.MongoDBQueryPartitioningWhereTester; @@ -50,10 +51,10 @@ public List getTestOracleFactory() { return oracles; } - public enum MongoDBOracleFactory implements OracleFactory { + public enum MongoDBOracleFactory implements OracleFactory { QUERY_PARTITIONING { @Override - public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws Exception { + public TestOracle create(MongoDBGlobalState globalState) throws Exception { List oracles = new ArrayList<>(); oracles.add(new MongoDBQueryPartitioningWhereTester(globalState)); return new CompositeTestOracle(oracles, globalState); @@ -61,7 +62,7 @@ public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws }, DOCUMENT_REMOVAL { @Override - public TestOracle create(MongoDBProvider.MongoDBGlobalState globalState) throws Exception { + public TestOracle create(MongoDBGlobalState globalState) throws Exception { List oracles = new ArrayList<>(); oracles.add(new MongoDBDocumentRemovalTester(globalState)); return new CompositeTestOracle(oracles, globalState); From 8c15ec8c116e3ac070bbdac7db7391e80e1916c3 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Wed, 21 Jul 2021 16:18:17 +0800 Subject: [PATCH 003/856] Fix typos for dbmsSpecificOptions --- src/sqlancer/GlobalState.java | 10 +++---- src/sqlancer/Main.java | 4 +-- src/sqlancer/ProviderAdapter.java | 2 +- src/sqlancer/arangodb/ArangoDBProvider.java | 2 +- .../arangodb/gen/ArangoDBInsertGenerator.java | 2 +- .../ArangoDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/citus/CitusProvider.java | 4 +-- .../clickhouse/ClickHouseProvider.java | 2 +- .../gen/ClickHouseExpressionGenerator.java | 2 +- .../cockroachdb/CockroachDBProvider.java | 6 ++-- .../gen/CockroachDBIndexGenerator.java | 2 +- .../gen/CockroachDBTableGenerator.java | 4 +-- src/sqlancer/duckdb/DuckDBProvider.java | 8 +++--- .../duckdb/gen/DuckDBExpressionGenerator.java | 28 +++++++++---------- .../duckdb/gen/DuckDBIndexGenerator.java | 2 +- .../duckdb/gen/DuckDBTableGenerator.java | 12 ++++---- src/sqlancer/mongodb/MongoDBProvider.java | 2 +- src/sqlancer/mongodb/MongoDBSchema.java | 2 +- .../mongodb/gen/MongoDBConstantGenerator.java | 2 +- .../mongodb/gen/MongoDBInsertGenerator.java | 2 +- .../gen/MongoDBMatchExpressionGenerator.java | 4 +-- .../mongodb/gen/MongoDBTableGenerator.java | 2 +- .../test/MongoDBDocumentRemovalBase.java | 4 +-- .../test/MongoDBQueryPartitioningBase.java | 4 +-- .../MongoDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/mysql/MySQLGlobalState.java | 2 +- src/sqlancer/postgres/PostgresProvider.java | 6 ++-- .../gen/PostgresExpressionGenerator.java | 2 +- .../postgres/gen/PostgresInsertGenerator.java | 2 +- src/sqlancer/sqlite3/SQLite3Provider.java | 14 +++++----- .../sqlite3/gen/SQLite3ColumnBuilder.java | 6 ++-- .../gen/SQLite3ExpressionGenerator.java | 12 ++++---- .../gen/ddl/SQLite3TableGenerator.java | 12 ++++---- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 2 +- .../sqlite3/oracle/SQLite3Fuzzer.java | 2 +- 35 files changed, 88 insertions(+), 88 deletions(-) diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 642c9da6a..64b5c731e 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -10,7 +10,7 @@ public abstract class GlobalState, S extends Ab protected C databaseConnection; private Randomly r; private MainOptions options; - private O dmbsSpecificOptions; + private O dbmsSpecificOptions; private S schema; private Main.StateLogger logger; private StateToReproduce state; @@ -26,12 +26,12 @@ public C getConnection() { } @SuppressWarnings("unchecked") - public void setDmbsSpecificOptions(Object dmbsSpecificOptions) { - this.dmbsSpecificOptions = (O) dmbsSpecificOptions; + public void setDbmsSpecificOptions(Object dbmsSpecificOptions) { + this.dbmsSpecificOptions = (O) dbmsSpecificOptions; } - public O getDmbsSpecificOptions() { - return dmbsSpecificOptions; + public O getDbmsSpecificOptions() { + return dbmsSpecificOptions; } public void setRandomly(Randomly r) { diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 4c1130f28..20024e437 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -309,7 +309,7 @@ public void run() throws Exception { state.setRandomly(r); state.setDatabaseName(databaseName); state.setMainOptions(options); - state.setDmbsSpecificOptions(command); + state.setDbmsSpecificOptions(command); try (C con = provider.createDatabase(state)) { QueryManager manager = new QueryManager<>(state); try { @@ -343,7 +343,7 @@ private G getInitializedGlobalState(long seed) { state.setRandomly(r); state.setDatabaseName(databaseName); state.setMainOptions(options); - state.setDmbsSpecificOptions(command); + state.setDbmsSpecificOptions(command); return state; } diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 1e3158563..a16f8388c 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -63,7 +63,7 @@ public void generateAndTestDatabase(G globalState) throws Exception { protected abstract void checkViewsAreValid(G globalState); protected TestOracle getTestOracle(G globalState) throws Exception { - List> testOracleFactory = globalState.getDmbsSpecificOptions() + List> testOracleFactory = globalState.getDbmsSpecificOptions() .getTestOracleFactory(); boolean testOracleRequiresMoreThanZeroRows = testOracleFactory.stream() .anyMatch(p -> p.requiresAllTablesToContainRows()); diff --git a/src/sqlancer/arangodb/ArangoDBProvider.java b/src/sqlancer/arangodb/ArangoDBProvider.java index d63d4a1cc..a44c4e422 100644 --- a/src/sqlancer/arangodb/ArangoDBProvider.java +++ b/src/sqlancer/arangodb/ArangoDBProvider.java @@ -47,7 +47,7 @@ private static int mapActions(ArangoDBGlobalState globalState, Action a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case CREATE_INDEX: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumberIndexes); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumberIndexes); default: throw new AssertionError(a); } diff --git a/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java index 9a27ccd57..3cfceeed4 100644 --- a/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java +++ b/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java @@ -26,7 +26,7 @@ private ArangoDBQueryAdapter generate() { ArangoDBConstantGenerator constantGenerator = new ArangoDBConstantGenerator(globalState); for (int i = 0; i < table.getColumns().size(); i++) { - if (!globalState.getDmbsSpecificOptions().testRandomTypeInserts) { + if (!globalState.getDbmsSpecificOptions().testRandomTypeInserts) { constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), table.getColumns().get(i).getType()); } else { diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java index 4484da8f7..6ad19fabf 100644 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java @@ -35,7 +35,7 @@ public void check() throws Exception { thirdResultSet.addAll(secondResultSet); assumeResultSetsAreEqual(firstResultSet, thirdResultSet, query); - if (state.getDmbsSpecificOptions().withOptimizerRuleTests) { + if (state.getDbmsSpecificOptions().withOptimizerRuleTests) { select.setFilterClause(predicate); query = ArangoDBVisitor.asSelectQuery(select); query.excludeRandomOptRules(); diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 49a548298..6d7e5e507 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -300,7 +300,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws Exception { @Override protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { - List oracles = ((CitusOptions) globalState.getDmbsSpecificOptions()).citusOracle.stream().map(o -> { + List oracles = ((CitusOptions) globalState.getDbmsSpecificOptions()).citusOracle.stream().map(o -> { try { return o.create(globalState); } catch (Exception e1) { @@ -415,7 +415,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE // reconnect to coordinator node, test database con = new SQLConnection(DriverManager.getConnection("jdbc:" + testURL, username, password)); ((CitusGlobalState) globalState) - .setRepartition(((CitusOptions) globalState.getDmbsSpecificOptions()).repartition); + .setRepartition(((CitusOptions) globalState.getDbmsSpecificOptions()).repartition); globalState.getState().commentStatements(); return con; } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 297369ded..d936238d4 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -112,7 +112,7 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ port = ClickHouseOptions.DEFAULT_PORT; } - ClickHouseOptions clickHouseOptions = globalState.getDmbsSpecificOptions(); + ClickHouseOptions clickHouseOptions = globalState.getDbmsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); String url = String.format("jdbc:clickhouse://%s:%d/default", host, port); String databaseName = globalState.getDatabaseName(); diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 3bb29e75d..1054055f9 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -91,7 +91,7 @@ protected ClickHouseLancerDataType getRandomType() { public List getRandomJoinClauses( List tables) { List joinStatements = new ArrayList<>(); - if (!globalState.getDmbsSpecificOptions().testJoins) { + if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } if (Randomly.getBoolean() && tables.size() > 1) { diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 8b675459e..ef9e56e98 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -135,10 +135,10 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio standardSettings.add("SET CLUSTER SETTING sql.stats.automatic_collection.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING timeseries.storage.enabled = 'off'"); - if (globalState.getDmbsSpecificOptions().testHashIndexes) { + if (globalState.getDbmsSpecificOptions().testHashIndexes) { standardSettings.add("set experimental_enable_hash_sharded_indexes='on';"); } - if (globalState.getDmbsSpecificOptions().testTempTables) { + if (globalState.getDbmsSpecificOptions().testTempTables) { standardSettings.add("SET experimental_enable_temp_tables = 'on'"); } for (String s : standardSettings) { @@ -241,7 +241,7 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio } total--; } - if (globalState.getDmbsSpecificOptions().makeVectorizationMoreLikely && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().makeVectorizationMoreLikely && Randomly.getBoolean()) { manager.execute(new SQLQueryAdapter("SET vectorize=on;")); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index fd4dfd70d..a37f5a26b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -38,7 +38,7 @@ public void buildStatement() { sb.append(table.getName()); List columns = table.getRandomNonEmptyColumnSubset(); addColumns(sb, columns, true); - boolean hashSharded = globalState.getDmbsSpecificOptions().testHashIndexes + boolean hashSharded = globalState.getDbmsSpecificOptions().testHashIndexes && Randomly.getBooleanWithSmallProbability(); if (hashSharded) { sb.append(" USING HASH WITH BUCKET_COUNT="); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index a33b7521f..d244d2080 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -35,13 +35,13 @@ public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { @Override public void buildStatement() { errors.add("and thus is not indexable"); // array types are not indexable - if (globalState.getDmbsSpecificOptions().testTempTables) { + if (globalState.getDbmsSpecificOptions().testTempTables) { errors.add("constraints on temporary tables may reference only temporary tables"); errors.add("constraints on permanent tables may reference only permanent tables"); } String tableName = globalState.getSchema().getFreeTableName(); sb.append("CREATE "); - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().testTempTables) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testTempTables) { sb.append("TEMP "); } sb.append("TABLE "); diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 3acfc56cb..73adb3cb0 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -65,20 +65,20 @@ private static int mapActions(DuckDBGlobalState globalState, Action a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case CREATE_INDEX: - if (!globalState.getDmbsSpecificOptions().testIndexes) { + if (!globalState.getDbmsSpecificOptions().testIndexes) { return 0; } // fall through case UPDATE: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumUpdates + 1); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); case VACUUM: // seems to be ignored case ANALYZE: // seems to be ignored case EXPLAIN: return r.getInteger(0, 2); case DELETE: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumDeletes + 1); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); case CREATE_VIEW: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumViews + 1); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); } diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 99446ad41..f907e62f2 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -51,28 +51,28 @@ protected Node generateExpression(int depth) { return new NewFunctionNode<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - if (!globalState.getDmbsSpecificOptions().testCollate) { + if (!globalState.getDbmsSpecificOptions().testCollate) { possibleOptions.remove(Expression.COLLATE); } - if (!globalState.getDmbsSpecificOptions().testFunctions) { + if (!globalState.getDbmsSpecificOptions().testFunctions) { possibleOptions.remove(Expression.FUNC); } - if (!globalState.getDmbsSpecificOptions().testCasts) { + if (!globalState.getDbmsSpecificOptions().testCasts) { possibleOptions.remove(Expression.CAST); } - if (!globalState.getDmbsSpecificOptions().testBetween) { + if (!globalState.getDbmsSpecificOptions().testBetween) { possibleOptions.remove(Expression.BETWEEN); } - if (!globalState.getDmbsSpecificOptions().testIn) { + if (!globalState.getDbmsSpecificOptions().testIn) { possibleOptions.remove(Expression.IN); } - if (!globalState.getDmbsSpecificOptions().testCase) { + if (!globalState.getDbmsSpecificOptions().testCase) { possibleOptions.remove(Expression.CASE); } - if (!globalState.getDmbsSpecificOptions().testBinaryComparisons) { + if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { possibleOptions.remove(Expression.BINARY_COMPARISON); } - if (!globalState.getDmbsSpecificOptions().testBinaryLogicals) { + if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { possibleOptions.remove(Expression.BINARY_LOGICAL); } Expression expr = Randomly.fromList(possibleOptions); @@ -135,32 +135,32 @@ public Node generateConstant() { DuckDBDataType type = DuckDBDataType.getRandom(); switch (type) { case INT: - if (!globalState.getDmbsSpecificOptions().testIntConstants) { + if (!globalState.getDbmsSpecificOptions().testIntConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createIntConstant(globalState.getRandomly().getInteger()); case DATE: - if (!globalState.getDmbsSpecificOptions().testDateConstants) { + if (!globalState.getDbmsSpecificOptions().testDateConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createDateConstant(globalState.getRandomly().getInteger()); case TIMESTAMP: - if (!globalState.getDmbsSpecificOptions().testTimestampConstants) { + if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); case VARCHAR: - if (!globalState.getDmbsSpecificOptions().testStringConstants) { + if (!globalState.getDbmsSpecificOptions().testStringConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createStringConstant(globalState.getRandomly().getString()); case BOOLEAN: - if (!globalState.getDmbsSpecificOptions().testBooleanConstants) { + if (!globalState.getDbmsSpecificOptions().testBooleanConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createBooleanConstant(Randomly.getBoolean()); case FLOAT: - if (!globalState.getDmbsSpecificOptions().testFloatConstants) { + if (!globalState.getDbmsSpecificOptions().testFloatConstants) { throw new IgnoreMeException(); } return DuckDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 4b6b47612..eec5384aa 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -50,7 +50,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(DuckDBToStringVisitor.asString(expr)); } errors.add("already exists!"); - if (globalState.getDmbsSpecificOptions().testRowid) { + if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("Cannot create an index on the rowid!"); } return new SQLQueryAdapter(sb.toString(), errors, true); diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index de48e5c0f..bb8fe2304 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -36,33 +36,33 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType()); - if (globalState.getDmbsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() && columns.get(i).getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { sb.append(" COLLATE "); sb.append(getRandomCollate()); } - if (globalState.getDmbsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" UNIQUE"); } - if (globalState.getDmbsSpecificOptions().testNotNullConstraints + if (globalState.getDbmsSpecificOptions().testNotNullConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); } - if (globalState.getDmbsSpecificOptions().testCheckConstraints + if (globalState.getDbmsSpecificOptions().testCheckConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" CHECK("); sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); DuckDBErrors.addExpressionErrors(errors); sb.append(")"); } - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().testDefaultValues) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); DuckDBErrors.addExpressionErrors(errors); sb.append(")"); } } - if (globalState.getDmbsSpecificOptions().testIndexes && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { errors.add("Invalid type for index"); List primaryKeyColumns = Randomly.nonEmptySubset(columns); sb.append(", PRIMARY KEY("); diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java index 5ff549d32..3b090dade 100644 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ b/src/sqlancer/mongodb/MongoDBProvider.java @@ -49,7 +49,7 @@ public static int mapActions(MongoDBGlobalState globalState, Action a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case CREATE_INDEX: - return r.getInteger(0, globalState.getDmbsSpecificOptions().maxNumberIndexes); + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumberIndexes); default: throw new AssertionError(a); } diff --git a/src/sqlancer/mongodb/MongoDBSchema.java b/src/sqlancer/mongodb/MongoDBSchema.java index e9a0afb99..5ae3cdd24 100644 --- a/src/sqlancer/mongodb/MongoDBSchema.java +++ b/src/sqlancer/mongodb/MongoDBSchema.java @@ -36,7 +36,7 @@ public BsonType getBsonType() { public static MongoDBDataType getRandom(MongoDBGlobalState state) { Set valueSet = new HashSet<>(Arrays.asList(values())); - if (state.getDmbsSpecificOptions().nullSafety) { + if (state.getDbmsSpecificOptions().nullSafety) { valueSet.remove(STRING); } MongoDBDataType[] configuredValues = new MongoDBDataType[valueSet.size()]; diff --git a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java index e81291543..2e6b15048 100644 --- a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java @@ -48,7 +48,7 @@ public void addRandomConstant(Document document, String key) { public void addRandomConstantWithType(Document document, String key, MongoDBDataType option) { MongoDBConstant constant; - if (globalState.getDmbsSpecificOptions().testNullInserts && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().testNullInserts && Randomly.getBooleanWithSmallProbability()) { constant = new MongoDBNullConstant(); constant.setValueInDocument(document, key); return; diff --git a/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java b/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java index 4501971b4..f8b8b3ffd 100644 --- a/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java @@ -25,7 +25,7 @@ public MongoDBQueryAdapter generate() { MongoDBConstantGenerator constantGenerator = new MongoDBConstantGenerator(globalState); for (int i = 0; i < table.getColumns().size(); i++) { - if (!globalState.getDmbsSpecificOptions().testRandomTypes) { + if (!globalState.getDbmsSpecificOptions().testRandomTypes) { constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), table.getColumns().get(i).getType()); } else { diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java index 0645d7f24..3e6c833d3 100644 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java @@ -44,7 +44,7 @@ public MongoDBMatchExpressionGenerator(MongoDBGlobalState globalState) { @Override public Node generateLeafNode() { List possibleOptions = new ArrayList<>(Arrays.asList(LeafExpression.values())); - if (!globalState.getDmbsSpecificOptions().testWithRegex) { + if (!globalState.getDbmsSpecificOptions().testWithRegex) { possibleOptions.remove(LeafExpression.REGEX); } LeafExpression expr = Randomly.fromList(possibleOptions); @@ -97,7 +97,7 @@ public Node generateConstant() { public Node generateConstant(MongoDBDataType type) { MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); - if (Randomly.getBooleanWithSmallProbability() && !globalState.getDmbsSpecificOptions().nullSafety) { + if (Randomly.getBooleanWithSmallProbability() && !globalState.getDbmsSpecificOptions().nullSafety) { return MongoDBConstant.createNullConstant(); } return generator.generateConstantWithType(type); diff --git a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java index 6a4f33d38..0153a6334 100644 --- a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java @@ -28,7 +28,7 @@ public MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { for (int i = 0; i < Randomly.smallNumber() + 1; i++) { String columnName = String.format("c%d", i); MongoDBDataType type = createColumn(columnName); - if (globalState.getDmbsSpecificOptions().testValidation) { + if (globalState.getDbmsSpecificOptions().testValidation) { createTableQuery.addValidation(columnName, type.getBsonType()); } } diff --git a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java index e3e3df17c..6faae42bc 100644 --- a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java +++ b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java @@ -44,7 +44,7 @@ public void check() throws Exception { } else { select.setLookupList(Randomly.nonEmptySubset(targetColumns)); } - if (state.getDmbsSpecificOptions().testComputedValues) { + if (state.getDbmsSpecificOptions().testComputedValues) { generateComputedColumns(); } } @@ -55,7 +55,7 @@ private void generateTargetColumns() { targetColumns.add(new MongoDBColumnTestReference(c, true)); } List joinsOtherTables = new ArrayList<>(); - if (!state.getDmbsSpecificOptions().nullSafety) { + if (!state.getDbmsSpecificOptions().nullSafety) { for (int i = 1; i < targetTables.getTables().size(); i++) { MongoDBSchema.MongoDBTable procTable = targetTables.getTables().get(i); for (MongoDBSchema.MongoDBColumn c : procTable.getColumns()) { diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java index b26aa6028..30ebe56bd 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -47,7 +47,7 @@ public void check() throws Exception { } else { select.setLookupList(Randomly.nonEmptySubset(targetColumns)); } - if (state.getDmbsSpecificOptions().testComputedValues) { + if (state.getDbmsSpecificOptions().testComputedValues) { generateComputedColumns(); } } @@ -69,7 +69,7 @@ private void generateTargetColumns() { targetColumns.add(new MongoDBColumnTestReference(c, true)); } List joinsOtherTables = new ArrayList<>(); - if (!state.getDmbsSpecificOptions().nullSafety) { + if (!state.getDbmsSpecificOptions().nullSafety) { for (int i = 1; i < targetTables.getTables().size(); i++) { MongoDBTable procTable = targetTables.getTables().get(i); for (MongoDBColumn c : procTable.getColumns()) { diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java index 5a7507672..53f625d9b 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java @@ -33,7 +33,7 @@ public void check() throws Exception { q = new MongoDBSelectQuery(select); List thirdResultSet = getResultSetAsDocumentList(q, state); - if (state.getDmbsSpecificOptions().testWithCount) { + if (state.getDbmsSpecificOptions().testWithCount) { select.setWithCountClause(true); select.setFilterClause(predicate); q = new MongoDBSelectQuery(select); diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 041ac440b..86cf6dd18 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -14,7 +14,7 @@ protected MySQLSchema readSchema() throws SQLException { } public boolean usesPQS() { - return getDmbsSpecificOptions().oracles.stream().anyMatch(o -> o == MySQLOracleFactory.PQS); + return getDbmsSpecificOptions().oracles.stream().anyMatch(o -> o == MySQLOracleFactory.PQS); } } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 683ba95d1..36dd55465 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -195,7 +195,7 @@ public void generateDatabase(PostgresGlobalState globalState) throws Exception { @Override public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLException { - if (globalState.getDmbsSpecificOptions().getTestOracleFactory().stream() + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch((o) -> o == PostgresOracleFactory.PQS)) { generateOnlyKnown = true; } @@ -205,7 +205,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE host = globalState.getOptions().getHost(); port = globalState.getOptions().getPort(); entryPath = "/test"; - entryURL = globalState.getDmbsSpecificOptions().connectionURL; + entryURL = globalState.getDbmsSpecificOptions().connectionURL; // trim URL to exclude "jdbc:" if (entryURL.startsWith("jdbc:")) { entryURL = entryURL.substring(5); @@ -305,7 +305,7 @@ protected void prepareTables(PostgresGlobalState globalState) throws Exception { private String getCreateDatabaseCommand(PostgresGlobalState state) { StringBuilder sb = new StringBuilder(); sb.append("CREATE DATABASE " + databaseName + " "); - if (Randomly.getBoolean() && ((PostgresOptions) state.getDmbsSpecificOptions()).testCollations) { + if (Randomly.getBoolean() && ((PostgresOptions) state.getDbmsSpecificOptions()).testCollations) { if (Randomly.getBoolean()) { sb.append("WITH ENCODING '"); sb.append(Randomly.fromOptions("utf8")); diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 010bb8fec..f979e97da 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -379,7 +379,7 @@ private PostgresExpression generateTextExpression(int depth) { if (expectedResult) { validOptions.remove(TextExpression.COLLATE); } - if (!globalState.getDmbsSpecificOptions().testCollations) { + if (!globalState.getDbmsSpecificOptions().testCollations) { validOptions.remove(TextExpression.COLLATE); } option = Randomly.fromList(validOptions); diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index f23387977..41c017b2e 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -49,7 +49,7 @@ public static SQLQueryAdapter insert(PostgresGlobalState globalState) { } sb.append(" VALUES"); - if (globalState.getDmbsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { StringBuilder sbRowValue = new StringBuilder(); sbRowValue.append("("); for (int i = 0; i < columns.size(); i++) { diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 35e9b90cd..54f1a6db4 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -168,7 +168,7 @@ private static int mapActions(SQLite3GlobalState globalState, Action a) { public void generateDatabase(SQLite3GlobalState globalState) throws Exception { Randomly r = new Randomly(SQLite3SpecialStringGenerator::generate); globalState.setRandomly(r); - if (globalState.getDmbsSpecificOptions().generateDatabase) { + if (globalState.getDbmsSpecificOptions().generateDatabase) { addSensiblePragmaDefaults(globalState); int nrTablesToCreate = 1; @@ -186,7 +186,7 @@ public void generateDatabase(SQLite3GlobalState globalState) throws Exception { } while (globalState.getSchema().getDatabaseTables().size() < nrTablesToCreate); assert globalState.getSchema().getTables().getTables().size() == nrTablesToCreate; checkTablesForGeneratedColumnLoops(globalState); - if (globalState.getDmbsSpecificOptions().testDBStats && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().testDBStats && Randomly.getBooleanWithSmallProbability()) { SQLQueryAdapter tableQuery = new SQLQueryAdapter( "CREATE VIRTUAL TABLE IF NOT EXISTS stat USING dbstat(main)"); globalState.executeStatement(tableQuery); @@ -224,10 +224,10 @@ private void checkTablesForGeneratedColumnLoops(SQLite3GlobalState globalState) private SQLQueryAdapter getTableQuery(SQLite3GlobalState globalState, int i) throws AssertionError { SQLQueryAdapter tableQuery; List options = new ArrayList<>(Arrays.asList(TableType.values())); - if (!globalState.getDmbsSpecificOptions().testFts) { + if (!globalState.getDbmsSpecificOptions().testFts) { options.remove(TableType.FTS); } - if (!globalState.getDmbsSpecificOptions().testRtree) { + if (!globalState.getDbmsSpecificOptions().testRtree) { options.remove(TableType.RTREE); } switch (Randomly.fromList(options)) { @@ -255,11 +255,11 @@ private void addSensiblePragmaDefaults(SQLite3GlobalState globalState) throws Ex if (!Randomly.getBooleanWithSmallProbability()) { pragmasToExecute.addAll(DEFAULT_PRAGMAS); } - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { // the PQS implementation currently assumes the default behavior of LIKE pragmasToExecute.add("PRAGMA case_sensitive_like=ON;"); } - if (Randomly.getBoolean() && globalState.getDmbsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().oracles != SQLite3OracleFactory.PQS) { // the encoding has an influence how binary strings are cast pragmasToExecute.add(String.format("PRAGMA encoding = '%s';", Randomly.fromOptions("UTF-8", "UTF-16", "UTF-16le", "UTF-16be"))); @@ -276,7 +276,7 @@ public SQLConnection createDatabase(SQLite3GlobalState globalState) throws SQLEx dir.mkdir(); } File dataBase = new File(dir, globalState.getDatabaseName() + ".db"); - if (dataBase.exists() && ((SQLite3GlobalState) globalState).getDmbsSpecificOptions().deleteIfExists) { + if (dataBase.exists() && ((SQLite3GlobalState) globalState).getDbmsSpecificOptions().deleteIfExists) { dataBase.delete(); } String url = "jdbc:sqlite:" + dataBase.getAbsolutePath(); diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index 41a6fdcef..e07533ed8 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -39,8 +39,8 @@ public boolean isContainsPrimaryKey() { } public String createColumn(String columnName, SQLite3GlobalState globalState, List columns) { - if (globalState.getDmbsSpecificOptions().oracles == SQLite3OracleFactory.PQS - || !globalState.getDmbsSpecificOptions().testCheckConstraints) { + if (globalState.getDbmsSpecificOptions().oracles == SQLite3OracleFactory.PQS + || !globalState.getDbmsSpecificOptions().testCheckConstraints) { allowCheck = false; } sb.append(columnName); @@ -51,7 +51,7 @@ public String createColumn(String columnName, SQLite3GlobalState globalState, Li if (Randomly.getBooleanWithRatherLowProbability()) { List constraints = Randomly.subset(Constraints.values()); if (!Randomly.getBooleanWithSmallProbability() - || globalState.getDmbsSpecificOptions().testGeneratedColumns) { + || globalState.getDbmsSpecificOptions().testGeneratedColumns) { constraints.remove(Constraints.GENERATED_AS); } if (constraints.contains(Constraints.GENERATED_AS)) { diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index edbbc29f6..e7480dc19 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -136,7 +136,7 @@ public List generateOrderBys() { public List getRandomJoinClauses(List tables) { List joinStatements = new ArrayList<>(); - if (!globalState.getDmbsSpecificOptions().testJoins) { + if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } if (Randomly.getBoolean() && tables.size() > 1) { @@ -165,7 +165,7 @@ public SQLite3Expression generateOrderingTerm() { if (Randomly.getBoolean()) { expr = new SQLite3OrderingTerm(expr, Ordering.getRandomValue()); } - if (globalState.getDmbsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { expr = new SQLite3PostfixText(expr, Randomly.fromOptions(" NULLS FIRST", " NULLS LAST"), null /* expr.getExpectedValue() */) { @Override @@ -253,13 +253,13 @@ public SQLite3Expression getRandomExpression(int depth) { if (!allowSubqueries) { list.remove(ExpressionType.RANDOM_QUERY); } - if (!globalState.getDmbsSpecificOptions().testFunctions) { + if (!globalState.getDbmsSpecificOptions().testFunctions) { list.remove(ExpressionType.FUNCTION); } - if (!globalState.getDmbsSpecificOptions().testMatch) { + if (!globalState.getDbmsSpecificOptions().testMatch) { list.remove(ExpressionType.MATCH); } - if (!globalState.getDmbsSpecificOptions().testIn) { + if (!globalState.getDbmsSpecificOptions().testIn) { list.remove(ExpressionType.IN_OPERATOR); } ExpressionType randomExpressionType = Randomly.fromList(list); @@ -504,7 +504,7 @@ static AnyFunction getRandom(SQLite3GlobalState globalState) { private static List getAllFunctions(SQLite3GlobalState globalState) { List functions = new ArrayList<>(Arrays.asList(AnyFunction.values())); - if (!globalState.getDmbsSpecificOptions().testSoundex) { + if (!globalState.getDbmsSpecificOptions().testSoundex) { boolean removed = functions.removeIf(f -> f.name.equals("soundex")); if (!removed) { throw new IllegalStateException(); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index ef7d696cf..cc4731911 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -62,7 +62,7 @@ public static SQLQueryAdapter createTableStatement(String tableName, SQLite3Glob public void start() { sb.append("CREATE "); - if (globalState.getDmbsSpecificOptions().testTempTables && Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testTempTables && Randomly.getBoolean()) { tempTable = true; if (Randomly.getBoolean()) { sb.append("TEMP "); @@ -110,12 +110,12 @@ public void start() { } } - if (globalState.getDmbsSpecificOptions().testForeignKeys && Randomly.getBooleanWithSmallProbability()) { + if (globalState.getDbmsSpecificOptions().testForeignKeys && Randomly.getBooleanWithSmallProbability()) { addForeignKey(); } - if (globalState.getDmbsSpecificOptions().testCheckConstraints && globalState - .getDmbsSpecificOptions().oracles != SQLite3OracleFactory.PQS /* + if (globalState.getDbmsSpecificOptions().testCheckConstraints && globalState + .getDbmsSpecificOptions().oracles != SQLite3OracleFactory.PQS /* * we are currently lacking a parser to * read column definitions, and would * interpret a COLLATE in the check @@ -126,7 +126,7 @@ public void start() { } sb.append(")"); - if (globalState.getDmbsSpecificOptions().testWithoutRowids && containsPrimaryKey && !containsAutoIncrement + if (globalState.getDbmsSpecificOptions().testWithoutRowids && containsPrimaryKey && !containsAutoIncrement && Randomly.getBoolean()) { // see https://sqlite.org/withoutrowid.html sb.append(" WITHOUT ROWID"); @@ -151,7 +151,7 @@ private void addColumnConstraints(String s) { * @see https://www.sqlite.org/foreignkeys.html */ private void addForeignKey() { - assert globalState.getDmbsSpecificOptions().testForeignKeys; + assert globalState.getDbmsSpecificOptions().testForeignKeys; List foreignKeyColumns; if (Randomly.getBoolean()) { foreignKeyColumns = Arrays.asList(Randomly.fromList(columnNames)); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 400c66864..ffa3af6cd 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -50,7 +50,7 @@ public static SQLQueryAdapter generate(SQLite3GlobalState globalState) throws SQ SQLite3Expression randomQuery; do { randomQuery = SQLite3RandomQuerySynthesizer.generate(globalState, size); - } while (globalState.getDmbsSpecificOptions().oracles == SQLite3OracleFactory.PQS + } while (globalState.getDbmsSpecificOptions().oracles == SQLite3OracleFactory.PQS && !checkAffinity(randomQuery)); sb.append(SQLite3Visitor.asString(randomQuery)); return new SQLQueryAdapter(sb.toString(), errors, true); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index 429b22dd6..8e868dfd0 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -20,7 +20,7 @@ public void check() throws Exception { String s = SQLite3Visitor .asString(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + ";"; try { - if (globalState.getDmbsSpecificOptions().executeQuery) { + if (globalState.getDbmsSpecificOptions().executeQuery) { globalState.executeStatement(new SQLQueryAdapter(s)); globalState.getManager().incrementSelectQueryCount(); } From 83531015bea6d442650d83039929df10bc9d4db4 Mon Sep 17 00:00:00 2001 From: Hao Wu Date: Mon, 26 Jul 2021 09:47:16 +0800 Subject: [PATCH 004/856] reorder import packages --- src/sqlancer/arangodb/ArangoDBOptions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index f1759bccb..6e7a607f0 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -10,8 +10,8 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; -import sqlancer.arangodb.test.ArangoDBQueryPartitioningWhereTester; import sqlancer.arangodb.ArangoDBProvider.ArangoDBGlobalState; +import sqlancer.arangodb.test.ArangoDBQueryPartitioningWhereTester; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; From b820fcd6b93cf52b5cc369da70b8fa4c1422dad4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 26 Jul 2021 22:21:06 +0200 Subject: [PATCH 005/856] Adapt to a recent PostgreSQL change --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 05e1e62ba..243073034 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -235,6 +235,7 @@ private void createColumnConstraint(PostgresDataType type, boolean serial) { switch (c) { case NULL_OR_NOT_NULL: sb.append(Randomly.fromOptions("NOT NULL", "NULL")); + errors.add("conflicting NULL/NOT NULL declarations"); break; case UNIQUE: sb.append("UNIQUE"); From f6ddf1e7d4ac18958dd9ef249da6d3c9309b4794 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 26 Jul 2021 22:35:46 +0200 Subject: [PATCH 006/856] Adapt to the removal of the H2 COMPRESS_LOB option --- src/sqlancer/h2/H2SetGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/h2/H2SetGenerator.java b/src/sqlancer/h2/H2SetGenerator.java index 26211a93c..88833e1e9 100644 --- a/src/sqlancer/h2/H2SetGenerator.java +++ b/src/sqlancer/h2/H2SetGenerator.java @@ -21,7 +21,6 @@ private enum Option { BUILTIN_ALIAS_OVERRIDE((r) -> Randomly.fromOptions("TRUE", "FALSE")), // COLLATION((r) -> Randomly.fromOptions("OFF", "ENGLISH", "ENGLISH STRENGTH PRIMARY", "ENGLISH STRENGTH SECONDARY", "ENGLISH STRENGTH TERTIARY", "ENGLISH STRENGTH IDENTICAL")), // - COMPRESS_LOB((r) -> Randomly.fromOptions("NO", "LZF", "DEFLATE")), // DEFAULT_NULL_ORDERING((r) -> Randomly.fromOptions("LOW", "HIGH", "FIRST", "LAST")), // DEFAULT_TABLE_TYPE((r) -> Randomly.fromOptions("MEMORY", "CACHED")), // IGNORECASE((r) -> Randomly.fromOptions("TRUE", "FALSE")), // From 2b3722810a00f4e8ababa26c7627ff8631acab7b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 20:49:36 +0000 Subject: [PATCH 007/856] Bump slf4j-simple from 1.7.30 to 1.7.32 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.30 to 1.7.32. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/commits) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1848ef107..ae4212606 100644 --- a/pom.xml +++ b/pom.xml @@ -254,7 +254,7 @@ org.slf4j slf4j-simple - 1.7.30 + 1.7.32 ru.yandex.clickhouse From 7adeaabb520d50bda0e04fa4cf33d3dc7d89abf2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 21:02:47 +0000 Subject: [PATCH 008/856] Bump clickhouse-jdbc from 0.3.1 to 0.3.1-patch Bumps [clickhouse-jdbc](https://github.com/ClickHouse/clickhouse-jdbc) from 0.3.1 to 0.3.1-patch. - [Release notes](https://github.com/ClickHouse/clickhouse-jdbc/releases) - [Changelog](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/ClickHouse/clickhouse-jdbc/compare/v0.3.1...v0.3.1-patch) --- updated-dependencies: - dependency-name: ru.yandex.clickhouse:clickhouse-jdbc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ae4212606..99e5796a0 100644 --- a/pom.xml +++ b/pom.xml @@ -259,7 +259,7 @@ ru.yandex.clickhouse clickhouse-jdbc - 0.3.1 + 0.3.1-patch com.h2database From 53edb4d639b9c09a2683cb5a279aeace38b8c4fd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 21:21:32 +0000 Subject: [PATCH 009/856] Bump maven-javadoc-plugin from 3.2.0 to 3.3.0 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.2.0...maven-javadoc-plugin-3.3.0) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99e5796a0..dbeceb225 100644 --- a/pom.xml +++ b/pom.xml @@ -338,7 +338,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.2.0 + 3.3.0 8 From 5bb9de2ca04938654382293118adbd1c2a883301 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 21:53:23 +0000 Subject: [PATCH 010/856] Bump postgresql from 42.2.20 to 42.2.23 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.20 to 42.2.23. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/REL42.2.23/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.20...REL42.2.23) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dbeceb225..8f8852a9f 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ org.postgresql postgresql - 42.2.20 + 42.2.23 org.xerial From 2171065193c1c9ff4ad171dd3bfc1cd2cfc7054d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 22:56:21 +0000 Subject: [PATCH 011/856] Bump junit-jupiter-engine from 5.7.1 to 5.7.2 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.7.1 to 5.7.2. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.7.1...r5.7.2) Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f8852a9f..1f9078b87 100644 --- a/pom.xml +++ b/pom.xml @@ -248,7 +248,7 @@ org.junit.jupiter junit-jupiter-engine - 5.7.1 + 5.7.2 test From 16a372b3b2fe456e514a1152c189f125aa2ff008 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Jul 2021 23:13:33 +0000 Subject: [PATCH 012/856] Bump mysql-connector-java from 8.0.25 to 8.0.26 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.25 to 8.0.26. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.25...8.0.26) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1f9078b87..97d742dd2 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ mysql mysql-connector-java - 8.0.25 + 8.0.26 org.mariadb.jdbc From 049d3bc3786e563a7afd1932c14d4570002ebf33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 08:18:21 +0000 Subject: [PATCH 013/856] Bump sqlite-jdbc from 3.34.0 to 3.36.0.1 Bumps [sqlite-jdbc](https://github.com/xerial/sqlite-jdbc) from 3.34.0 to 3.36.0.1. - [Release notes](https://github.com/xerial/sqlite-jdbc/releases) - [Changelog](https://github.com/xerial/sqlite-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/xerial/sqlite-jdbc/compare/3.34.0...3.36.0.1) --- updated-dependencies: - dependency-name: org.xerial:sqlite-jdbc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97d742dd2..c860e0690 100644 --- a/pom.xml +++ b/pom.xml @@ -228,7 +228,7 @@ org.xerial sqlite-jdbc - 3.34.0 + 3.36.0.1 mysql From 063a04c56eabce449ed3bdc50d879b332064ca4d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Jul 2021 08:42:19 +0000 Subject: [PATCH 014/856] Bump formatter-maven-plugin from 2.15.0 to 2.16.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.15.0 to 2.16.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.15.0...formatter-maven-plugin-2.16.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c860e0690..54bd97bdd 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.15.0 + 2.16.0 eclipseformat From 6c08269ff48f4cf667490e8794bdcf830958813b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 28 Jul 2021 10:11:11 +0200 Subject: [PATCH 015/856] Fix an expected MySQL 'Row size too large' error --- src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index 408463f24..3f418385d 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -114,6 +114,7 @@ public SQLQueryAdapter create() { errors.add("out of range"); errors.add("Data truncated for functional index"); errors.add("used in key specification without a key length"); + errors.add("Row size too large"); // seems to happen together with MIN_ROWS in the table declaration return new SQLQueryAdapter(string, errors, true); } From 5fa10dfd581719296db0cb9737dbfac1a97f08d5 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Wed, 30 Jun 2021 16:52:39 -0700 Subject: [PATCH 016/856] Update CockroachDB to v21.1.7 --- .github/workflows/main.yml | 6 +++--- src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java | 1 + .../gen/CockroachDBSetClusterSettingGenerator.java | 1 - 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 61458f894..9c4adaa15 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -110,11 +110,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.2.8.linux-amd64.tgz | tar xvz - cd cockroach-v20.2.8.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v21.1.7.linux-amd64.tgz | tar xvz + cd cockroach-v21.1.7.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v20.2.8.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v21.1.7.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index a37f5a26b..9ded19411 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -27,6 +27,7 @@ public void buildStatement() { errors.add("violates unique constraint"); errors.add("schema change statement cannot follow a statement that has written in the same transaction"); errors.add("and thus is not indexable"); // array types are not indexable + errors.add("the following columns are not indexable due to their type"); // array types are not indexable errors.add("cannot determine type of empty array. Consider annotating with the desired type"); errors.add("incompatible IF expression"); // TODO: investigate; seems to be a bug CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 4fb36a5d1..2d411f26e 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -15,7 +15,6 @@ private CockroachDBSetClusterSettingGenerator() { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBClusterSetting { - COMPATOR_ENABLED("compactor.enabled", CockroachDBSetSessionGenerator::onOff), BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), From 78eaf96c2115d944ed523dfb8ca744a3beba8080 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Wed, 30 Jun 2021 16:53:11 -0700 Subject: [PATCH 017/856] Remove interleaved tables and indexes for CockroachDB Interleaved tables are disabled by default in CockroachDB v21.1, and will be permanently removed in a future release. This commit removes generation of interleaved tables and indexes. --- .../cockroachdb/CockroachDBProvider.java | 15 +++------------ .../cockroachdb/gen/CockroachDBGenerator.java | 19 ------------------- .../gen/CockroachDBIndexGenerator.java | 3 --- .../gen/CockroachDBTableGenerator.java | 3 --- .../gen/CockroachDBTruncateGenerator.java | 1 - 5 files changed, 3 insertions(+), 38 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index ef9e56e98..2fcdf1930 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -54,12 +54,9 @@ public enum Action { SHOW(CockroachDBShowGenerator::show), // TRANSACTION((g) -> { String s = Randomly.fromOptions("BEGIN", "ROLLBACK", "COMMIT"); - return new SQLQueryAdapter(s, - ExpectedErrors.from("there is no transaction in progress", - "there is already a transaction in progress", "current transaction is aborted", - "does not exist" /* interleaved indexes */)); - }), // - EXPLAIN((g) -> { + return new SQLQueryAdapter(s, ExpectedErrors.from("there is no transaction in progress", + "there is already a transaction in progress", "current transaction is aborted")); + }), EXPLAIN((g) -> { StringBuilder sb = new StringBuilder("EXPLAIN "); ExpectedErrors errors = new ExpectedErrors(); if (Randomly.getBoolean()) { @@ -267,12 +264,6 @@ public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws S globalState.getState().logStatement("USE " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); - } catch (SQLException e) { - if (e.getMessage().contains("ERROR: invalid interleave backreference")) { - throw new IgnoreMeException(); // TODO: investigate - } else { - throw e; - } } try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java index e06cc8d9d..34d0b9355 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBGenerator.java @@ -1,12 +1,10 @@ package sqlancer.cockroachdb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.common.gen.AbstractGenerator; public abstract class CockroachDBGenerator extends AbstractGenerator { @@ -32,21 +30,4 @@ static void addColumns(StringBuilder sb, List columns, boolea sb.append(")"); } - void generateInterleave() { - // TODO make this more likely to succeed - CockroachDBTable parentTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - List parentColumns = parentTable.getRandomNonEmptyColumnSubset(); - sb.append(" INTERLEAVE IN PARENT "); - sb.append(parentTable.getName()); - sb.append("("); - sb.append(parentColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - errors.add("must refer to a prefix of the primary key column names being interleaved"); - errors.add("must refer to a prefix of the index column names being interleaved"); - errors.add("must match the parent's primary index"); - errors.add("must match type and sort direction of the parent's primary index"); - errors.add("must be a prefix of the index columns being interleaved"); - errors.add("must be a prefix of the primary key columns being interleaved"); - } - } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 9ded19411..66c2936fd 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -53,9 +53,6 @@ public void buildStatement() { sb.append(" "); addColumns(sb, table.getRandomNonEmptyColumnSubset(), false); } - if (!hashSharded /* interleaved indexes cannot also be hash sharded */ && Randomly.getBoolean()) { - generateInterleave(); - } } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index d244d2080..720f2a300 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -167,9 +167,6 @@ public void buildStatement() { errors.add("there is no unique constraint matching given keys for referenced table"); } sb.append(")"); - if (Randomly.getBooleanWithRatherLowProbability() && !globalState.getSchema().getDatabaseTables().isEmpty()) { - generateInterleave(); - } errors.add("collatedstring"); CockroachDBErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index c5bdc2881..02579cf32 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -13,7 +13,6 @@ private CockroachDBTruncateGenerator() { // https://www.cockroachlabs.com/docs/v19.2/truncate.html public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); - errors.add("is interleaved by table"); errors.add("is referenced by foreign key"); // https://github.com/cockroachdb/cockroach/issues/47030 From 60666706e8621bec465410890401604d502ed771 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Aug 2021 11:47:03 +0000 Subject: [PATCH 018/856] Bump mariadb-java-client from 2.7.3 to 2.7.4 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.7.3 to 2.7.4. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/2.7.3...2.7.4) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 54bd97bdd..e932cfd93 100644 --- a/pom.xml +++ b/pom.xml @@ -238,7 +238,7 @@ org.mariadb.jdbc mariadb-java-client - 2.7.3 + 2.7.4 org.duckdb From f830e58f70d7de7f4f35c9f99e947766ecbfe5ff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Aug 2021 09:13:32 +0000 Subject: [PATCH 019/856] Bump sqlite-jdbc from 3.36.0.1 to 3.36.0.3 Bumps [sqlite-jdbc](https://github.com/xerial/sqlite-jdbc) from 3.36.0.1 to 3.36.0.3. - [Release notes](https://github.com/xerial/sqlite-jdbc/releases) - [Changelog](https://github.com/xerial/sqlite-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/xerial/sqlite-jdbc/compare/3.36.0.1...3.36.0.3) --- updated-dependencies: - dependency-name: org.xerial:sqlite-jdbc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e932cfd93..1cfa89d74 100644 --- a/pom.xml +++ b/pom.xml @@ -228,7 +228,7 @@ org.xerial sqlite-jdbc - 3.36.0.1 + 3.36.0.3 mysql From 33b8a7a07aeca158d925ac95c4c92e073e65c447 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Sep 2021 09:08:43 +0000 Subject: [PATCH 020/856] Bump maven-javadoc-plugin from 3.3.0 to 3.3.1 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.3.0 to 3.3.1. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.3.0...maven-javadoc-plugin-3.3.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1cfa89d74..20f8088dc 100644 --- a/pom.xml +++ b/pom.xml @@ -338,7 +338,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.3.0 + 3.3.1 8 From 92b75a619352967853f68e5a49b4c740562dd42e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Sep 2021 09:10:44 +0000 Subject: [PATCH 021/856] Bump junit-jupiter-engine from 5.7.2 to 5.8.0 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.7.2 to 5.8.0. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.7.2...r5.8.0) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 20f8088dc..c506e2940 100644 --- a/pom.xml +++ b/pom.xml @@ -248,7 +248,7 @@ org.junit.jupiter junit-jupiter-engine - 5.7.2 + 5.8.0 test From 357cd4414185b98c0746459799d652e53ae2186a Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 16 Sep 2021 14:18:50 +0300 Subject: [PATCH 022/856] Fix incorrect variable names in PostgresGlobalState --- src/sqlancer/postgres/PostgresGlobalState.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index f36536975..119d9fc24 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -40,16 +40,16 @@ public void setConnection(SQLConnection con) { } private List getCollnames(SQLConnection con) throws SQLException { - List opClasses = new ArrayList<>(); + List collNames = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s .executeQuery("SELECT collname FROM pg_collation WHERE collname LIKE '%utf8' or collname = 'C';")) { while (rs.next()) { - opClasses.add(rs.getString(1)); + collNames.add(rs.getString(1)); } } } - return opClasses; + return collNames; } private List getOpclasses(SQLConnection con) throws SQLException { @@ -65,15 +65,15 @@ private List getOpclasses(SQLConnection con) throws SQLException { } private List getOperators(SQLConnection con) throws SQLException { - List opClasses = new ArrayList<>(); + List operators = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { while (rs.next()) { - opClasses.add(rs.getString(1)); + operators.add(rs.getString(1)); } } } - return opClasses; + return operators; } public List getOperators() { From 749b0fd0345101b557bc0a63911c7dc001738c3a Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Wed, 15 Sep 2021 17:01:38 +0300 Subject: [PATCH 023/856] Add Citus 10.1 support --- .github/workflows/main.yml | 6 +++--- .travis.yml | 4 ++-- src/sqlancer/citus/CitusProvider.java | 6 +++--- src/sqlancer/citus/CitusSchema.java | 4 ++++ src/sqlancer/postgres/PostgresSchema.java | 1 + 5 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9c4adaa15..074a8702a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: - name: Set up Citus run: | curl https://install.citusdata.com/community/deb.sh | sudo bash - sudo apt-get -y install postgresql-13-citus-9.5 + sudo apt-get -y install postgresql-13-citus-10.1 sudo chown -R $USER:$USER /var/run/postgresql export PATH=/usr/lib/postgresql/13/bin:$PATH cd ~ @@ -68,8 +68,8 @@ jobs: psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test - psql -c "SELECT * from master_add_node('localhost', 9701);" -p 9700 -U $USER -d test - psql -c "SELECT * from master_add_node('localhost', 9702);" -p 9700 -U $USER -d test + psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test + psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test - name: Run Tests run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test diff --git a/.travis.yml b/.travis.yml index 13cf6704d..318ea6fa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -130,8 +130,8 @@ matrix: - psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test - psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test - psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test - - psql -c "SELECT * from master_add_node('localhost', 9701);" -p 9700 -U $USER -d test - - psql -c "SELECT * from master_add_node('localhost', 9702);" -p 9700 -U $USER -d test + - psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test + - psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test script: - CITUS_AVAILABLE=true mvn -Dtest=TestCitus test - name: ClickHouse diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 6d7e5e507..978168800 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -312,10 +312,10 @@ protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLEx private List readCitusWorkerNodes(PostgresGlobalState globalState, SQLConnection con) throws SQLException { - globalState.getState().logStatement("SELECT * FROM master_get_active_worker_nodes()"); + globalState.getState().logStatement("SELECT * FROM citus_get_active_worker_nodes()"); List citusWorkerNodes = new ArrayList<>(); try (Statement s = con.createStatement()) { - ResultSet rs = s.executeQuery("SELECT * FROM master_get_active_worker_nodes();"); + ResultSet rs = s.executeQuery("SELECT * FROM citus_get_active_worker_nodes();"); while (rs.next()) { String nodeHost = rs.getString("node_name"); int nodePort = rs.getInt("node_port"); @@ -374,7 +374,7 @@ private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List citusWorkerNodes) throws SQLException { for (CitusWorkerNode w : citusWorkerNodes) { - String addWorkers = "SELECT * from master_add_node('" + w.getHost() + "', " + w.getPort() + ");"; + String addWorkers = "SELECT * from citus_add_node('" + w.getHost() + "', " + w.getPort() + ");"; globalState.getState().logStatement(addWorkers); try (Statement s = con.createStatement()) { s.execute(addWorkers); diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index 694292a25..defda8804 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -65,6 +65,10 @@ public static CitusSchema fromConnection(SQLConnection con, String databaseName) "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { while (rs.next()) { String tableName = rs.getString("table_name"); + /* citus_tables is a helper view, we don't need to test with it so we let's ignore it */ + if (tableName.equals("citus_tables")) { + continue; + } String distributionColumnName = rs.getString("dist_col_name"); Integer colocationId = rs.getInt("colocationid"); if (rs.wasNull()) { diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 889b2e2a6..c99c8648e 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -124,6 +124,7 @@ public static PostgresDataType getColumnType(String typeString) { case "character": case "character varying": case "name": + case "regclass": return PostgresDataType.TEXT; case "numeric": return PostgresDataType.DECIMAL; From 076ec8e2eff85b448b3ba9db0ddaf0a8f19f76cf Mon Sep 17 00:00:00 2001 From: Neha George Date: Mon, 20 Sep 2021 09:06:42 -0400 Subject: [PATCH 024/856] Remove JOIN Oracle in CockroachDB This oracle does not necessarily produce the same results for both queries, thus rendering it an invalid TLP oracle. --- .../cockroachdb/CockroachDBOptions.java | 7 -- .../oracle/tlp/CockroachDBTLPJoinOracle.java | 118 ------------------ 2 files changed, 125 deletions(-) delete mode 100644 src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 2af62ae69..7673c4c0a 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -18,7 +18,6 @@ import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPJoinOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPWhereOracle; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; @@ -77,12 +76,6 @@ public TestOracle create(CockroachDBGlobalState globalState) throws SQLException return new CockroachDBTLPExtendedWhereOracle(globalState); } }, - JOIN { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPJoinOracle(globalState); - } - }, QUERY_PARTITIONING { @Override public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java deleted file mode 100644 index 988f82fba..000000000 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPJoinOracle.java +++ /dev/null @@ -1,118 +0,0 @@ -package sqlancer.cockroachdb.oracle.tlp; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.cockroachdb.CockroachDBErrors; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; -import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.cockroachdb.ast.CockroachDBColumnReference; -import sqlancer.cockroachdb.ast.CockroachDBConstant; -import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBJoin.OuterType; -import sqlancer.cockroachdb.ast.CockroachDBNotOperation; -import sqlancer.cockroachdb.ast.CockroachDBSelect; -import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; -import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class CockroachDBTLPJoinOracle implements TestOracle { - - final CockroachDBGlobalState state; - final ExpectedErrors errors = new ExpectedErrors(); - - CockroachDBSchema s; - CockroachDBTables targetTables; - CockroachDBExpressionGenerator gen; - CockroachDBSelect select; - CockroachDBExpression predicate; - CockroachDBExpression negatedPredicate; - CockroachDBExpression isNullPredicate; - - public CockroachDBTLPJoinOracle(CockroachDBGlobalState state) { - errors.add("GROUP BY term out of range"); - CockroachDBErrors.addExpressionErrors(errors); - this.state = state; - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new CockroachDBExpressionGenerator(state).setColumns(targetTables.getColumns()); - select = new CockroachDBSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - if (tables.size() < 2) { - return; - } - List tableList = tables.stream().map(t -> new CockroachDBTableReference(t)) - .collect(Collectors.toList()); - CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); - CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); - CockroachDBJoin leftJoinTrue = CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, - CockroachDBConstant.createBooleanConstant(true)); - - select.setJoinList(Arrays.asList(leftJoinTrue)); - select.setFromList(tableList); - select.setWhereClause(null); - predicate = generatePredicate(); - negatedPredicate = new CockroachDBNotOperation(predicate); - isNullPredicate = new CockroachDBUnaryPostfixOperation(predicate, CockroachDBUnaryPostfixOperator.IS_NULL); - - String originalQueryString1 = CockroachDBVisitor.asString(select); - - CockroachDBJoin leftJoinFalse = CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, - CockroachDBConstant.createBooleanConstant(false)); - select.setJoinList(Arrays.asList(leftJoinFalse)); - String originalQueryString2 = CockroachDBVisitor.asString(select); - String originalQueryString = originalQueryString1 + " UNION ALL " + originalQueryString2 + " UNION ALL " - + originalQueryString2; - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - // boolean allowOrderBy = Randomly.getBoolean(); - // if (allowOrderBy) { - // select.setOrderByExpressions(gen.getOrderingTerms()); - // } - - select.setJoinList( - Arrays.asList(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, predicate))); - String firstQueryString = CockroachDBVisitor.asString(select); - - select.setJoinList(Arrays - .asList(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, negatedPredicate))); - String secondQueryString = CockroachDBVisitor.asString(select); - - select.setJoinList( - Arrays.asList(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.LEFT, isNullPredicate))); - String thirdQueryString = CockroachDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - - List generateFetchColumns() { - return Arrays.asList(new CockroachDBColumnReference(targetTables.getColumns().get(0))); - } - - CockroachDBExpression generatePredicate() { - return gen.generateExpression(CockroachDBDataType.BOOL.get()); - } - -} From 17f8e4f770f5e7e5b26493f707ba9f820bc78acb Mon Sep 17 00:00:00 2001 From: Halil Ozan Akgul Date: Thu, 16 Sep 2021 12:32:14 +0300 Subject: [PATCH 025/856] Add support for access methods --- src/sqlancer/citus/CitusProvider.java | 16 ++++++++++++ src/sqlancer/citus/gen/CitusCommon.java | 3 +++ .../citus/gen/CitusTableGenerator.java | 6 +++++ .../postgres/PostgresGlobalState.java | 25 +++++++++++++++++++ .../postgres/gen/PostgresTableGenerator.java | 21 +++++++++++++++- 5 files changed, 70 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 978168800..172149a52 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -21,8 +21,10 @@ import sqlancer.citus.gen.CitusIndexGenerator; import sqlancer.citus.gen.CitusInsertGenerator; import sqlancer.citus.gen.CitusSetGenerator; +import sqlancer.citus.gen.CitusTableGenerator; import sqlancer.citus.gen.CitusUpdateGenerator; import sqlancer.citus.gen.CitusViewGenerator; +import sqlancer.common.DBMSCommon; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; @@ -271,6 +273,20 @@ private static void createDistributedTable(String tableName, CitusGlobalState gl distributeTable(columns, tableName, globalState); } + @Override + protected void createTables(PostgresGlobalState globalState, int numTables) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = CitusTableGenerator.generate(tableName, globalState.getSchema(), + generateOnlyKnown, globalState); + globalState.executeStatement(createTable); + } catch (IgnoreMeException e) { + + } + } + } + @Override public void generateDatabase(PostgresGlobalState globalState) throws Exception { readFunctions(globalState); diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 2b7bdaea5..f4a121efe 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -47,6 +47,9 @@ public static void addCitusErrors(ExpectedErrors errors) { errors.add("cannot pushdown the subquery"); // see https://github.com/sqlancer/sqlancer/issues/215 errors.add("direct joins between distributed and local tables are not supported"); + errors.add("unlogged columnar tables are not supported"); + errors.add("UPDATE and CTID scans not supported for ColumnarScan"); + errors.add("indexes not supported for columnar tables"); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java index 45d80e6aa..c340400f3 100644 --- a/src/sqlancer/citus/gen/CitusTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -1,5 +1,6 @@ package sqlancer.citus.gen; +import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.gen.PostgresTableGenerator; @@ -12,4 +13,9 @@ public CitusTableGenerator(String tableName, PostgresSchema newSchema, boolean g CitusCommon.addCitusErrors(errors); } + public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, + PostgresGlobalState globalState) { + return new CitusTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); + } + } diff --git a/src/sqlancer/postgres/PostgresGlobalState.java b/src/sqlancer/postgres/PostgresGlobalState.java index 119d9fc24..0f2c7ebdb 100644 --- a/src/sqlancer/postgres/PostgresGlobalState.java +++ b/src/sqlancer/postgres/PostgresGlobalState.java @@ -23,6 +23,7 @@ public class PostgresGlobalState extends SQLGlobalState operators = Collections.emptyList(); private List collates = Collections.emptyList(); private List opClasses = Collections.emptyList(); + private List tableAccessMethods = Collections.emptyList(); // store and allow filtering by function volatility classifications private final Map functionsAndTypes = new HashMap<>(); private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); @@ -34,6 +35,7 @@ public void setConnection(SQLConnection con) { this.opClasses = getOpclasses(getConnection()); this.operators = getOperators(getConnection()); this.collates = getCollnames(getConnection()); + this.tableAccessMethods = getTableAccessMethods(getConnection()); } catch (SQLException e) { throw new AssertionError(e); } @@ -76,6 +78,21 @@ private List getOperators(SQLConnection con) throws SQLException { return operators; } + private List getTableAccessMethods(SQLConnection con) throws SQLException { + List tableAccessMethods = new ArrayList<>(); + try (Statement s = con.createStatement()) { + /* + * pg_am includes both index and table access methods so we need to filter with amtype = 't' + */ + try (ResultSet rs = s.executeQuery("SELECT amname FROM pg_am WHERE amtype = 't';")) { + while (rs.next()) { + tableAccessMethods.add(rs.getString(1)); + } + } + } + return tableAccessMethods; + } + public List getOperators() { return operators; } @@ -100,6 +117,14 @@ public String getRandomOpclass() { return Randomly.fromList(opClasses); } + public List getTableAccessMethods() { + return tableAccessMethods; + } + + public String getRandomTableAccessMethod() { + return Randomly.fromList(tableAccessMethods); + } + @Override public PostgresSchema readSchema() throws SQLException { return PostgresSchema.fromConnection(getConnection(), getDatabaseName()); diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 243073034..7d65b8ebd 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -23,6 +23,7 @@ public class PostgresTableGenerator { private boolean columnHasPrimaryKey; private final StringBuilder sb = new StringBuilder(); private boolean isTemporaryTable; + private boolean isPartitionedTable; private final PostgresSchema newSchema; private final List columnsToBeAdded = new ArrayList<>(); protected final ExpectedErrors errors = new ExpectedErrors(); @@ -63,7 +64,7 @@ public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchem return new PostgresTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); } - private SQLQueryAdapter generate() { + protected SQLQueryAdapter generate() { columnCanHavePrimaryKey = true; sb.append("CREATE"); if (Randomly.getBoolean()) { @@ -111,6 +112,7 @@ private void createStandard() throws AssertionError { sb.append(")"); generateInherits(); generatePartitionBy(); + generateUsing(); PostgresCommon.generateWith(sb, globalState, errors); if (Randomly.getBoolean() && isTemporaryTable) { sb.append(" ON COMMIT "); @@ -152,8 +154,10 @@ private void createColumn(String name) throws AssertionError { private void generatePartitionBy() { if (Randomly.getBoolean()) { + isPartitionedTable = false; return; } + isPartitionedTable = true; sb.append(" PARTITION BY "); // TODO "RANGE", String partitionOption = Randomly.fromOptions("RANGE", "LIST", "HASH"); @@ -185,6 +189,21 @@ private void generatePartitionBy() { sb.append(")"); } + private void generateUsing() { + /* + * Postgres does not allow specifying USING clause for partitioned tables since they don't have any storage + * associated with them + */ + if (isPartitionedTable) { + return; + } + if (Randomly.getBoolean()) { + return; + } + sb.append(" USING "); + sb.append(globalState.getRandomTableAccessMethod()); + } + private void generateInherits() { if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { sb.append(" INHERITS("); From 8ec26f7b036e088f8f3dcf08bd3f17f1172315ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Sep 2021 09:10:12 +0000 Subject: [PATCH 026/856] Bump junit-jupiter-engine from 5.8.0 to 5.8.1 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.8.0 to 5.8.1. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.8.0...r5.8.1) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c506e2940..0d7b1878c 100644 --- a/pom.xml +++ b/pom.xml @@ -248,7 +248,7 @@ org.junit.jupiter junit-jupiter-engine - 5.8.0 + 5.8.1 test From 448989040cb09fa3d5787ce43e51f8014cf9dbd6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Sep 2021 09:12:33 +0000 Subject: [PATCH 027/856] Bump postgresql from 42.2.23 to 42.2.24 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.23 to 42.2.24. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/REL42.2.24/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.23...REL42.2.24) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0d7b1878c..1a943045f 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ org.postgresql postgresql - 42.2.23 + 42.2.24 org.xerial From 2cd5db3c00e103e8f63d9d7a47839e6f927acadc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 10 Nov 2021 19:08:39 +0100 Subject: [PATCH 028/856] Update and fix the Citus CI step This step started failing with an error "postgresql-13-citus-10.1 : Depends: postgresql-13 but it is not installable" --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 074a8702a..a6629c889 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,6 +43,7 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up Citus run: | + echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list curl https://install.citusdata.com/community/deb.sh | sudo bash sudo apt-get -y install postgresql-13-citus-10.1 sudo chown -R $USER:$USER /var/run/postgresql From 29d3e88a3a725c6f51ed2584b11701c681901e04 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 10 Nov 2021 20:06:08 +0100 Subject: [PATCH 029/856] Temporarily disable warnings due to the newly-appearing xml-apis warning Warning: The artifact xml-apis:xml-apis:jar:2.0.2 has been relocated to xml-apis:xml-apis:jar:1.0.b2 --- .github/workflows/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a6629c889..44ef6aebc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,8 +21,6 @@ jobs: uses: actions/setup-java@v1 with: java-version: 1.8 - - name: Build and check for warnings - run: mvn -B package -DskipTests 2>/dev/null | grep -vz "WARNING" - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests From 799cca36cb2f5142c16a17d82ed1d8438a9f473b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 20:41:11 +0000 Subject: [PATCH 030/856] Bump postgresql from 42.2.24 to 42.3.1 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.24 to 42.3.1. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.24...REL42.3.1) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1a943045f..8f231ce72 100644 --- a/pom.xml +++ b/pom.xml @@ -223,7 +223,7 @@ org.postgresql postgresql - 42.2.24 + 42.3.1 org.xerial From 107417e59224c6d5c7ec2f6b09a8c4657bb02897 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 20:53:42 +0000 Subject: [PATCH 031/856] Bump mysql-connector-java from 8.0.26 to 8.0.27 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.26 to 8.0.27. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.26...8.0.27) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8f231ce72..32ad7cd73 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ mysql mysql-connector-java - 8.0.26 + 8.0.27 org.mariadb.jdbc From bbb4ac677f2ca6d5e97afd8373b087386e5244c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 21:18:02 +0000 Subject: [PATCH 032/856] Bump spotbugs-maven-plugin from 4.2.2 to 4.4.2.2 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.2.2 to 4.4.2.2. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/4.2.2...spotbugs-maven-plugin-4.4.2.2) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 32ad7cd73..7ab1d96ab 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.2.2 + 4.4.2.2 spotbugs From 1cabe26e6c969b9eeb075b14e64e57cf3fef51b3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 20:53:39 +0000 Subject: [PATCH 033/856] Bump ecj from 3.25.0 to 3.27.0 Bumps ecj from 3.25.0 to 3.27.0. --- updated-dependencies: - dependency-name: org.eclipse.jdt:ecj dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7ab1d96ab..a47743728 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ org.eclipse.jdt ecj - 3.25.0 + 3.27.0 From 259bb3e2c87d3ea0dd5bb1e4de0d8fc3212666e1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 10 Nov 2021 22:05:54 +0100 Subject: [PATCH 034/856] Discontinue support for Java 1.8 to enable the ecj upgrade --- .github/workflows/main.yml | 62 ++++++++++++++------------------------ 1 file changed, 22 insertions(+), 40 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 44ef6aebc..ac460d4be 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,10 +17,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests @@ -33,10 +33,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up Citus @@ -79,10 +79,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up ClickHouse @@ -101,10 +101,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up CockroachDB @@ -125,10 +125,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build run: mvn -B package -DskipTests=true - name: DuckDB Tests @@ -141,10 +141,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up H2 @@ -166,10 +166,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Install MariaDB @@ -191,10 +191,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up MySQL @@ -226,10 +226,10 @@ jobs: postgresql user: 'sqlancer' postgresql password: 'sqlancer' postgresql db: 'test' - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -243,10 +243,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests @@ -262,10 +262,10 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up TiDB @@ -278,24 +278,6 @@ jobs: - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test - - java8: - name: Java 8 Compatibility (DuckDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build - run: mvn -B package -DskipTests=true - - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - java13: name: Java 13 Compatibility (DuckDB) runs-on: ubuntu-latest From 9452e22b42299301422c014590bcef8bc3b112da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Nov 2021 09:38:11 +0000 Subject: [PATCH 035/856] Bump formatter-maven-plugin from 2.16.0 to 2.17.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.16.0 to 2.17.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.16.0...formatter-maven-plugin-2.17.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a47743728..e75164b9a 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.16.0 + 2.17.0 eclipseformat From 1f781b275af40b9124b12a3557a9a94c147bbf26 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 17 Nov 2021 00:39:47 +0100 Subject: [PATCH 036/856] Remove the PostgreSQL vacuum_cleanup_index_scale_factor option. This option was removed with Postgres 14. Fixes https://github.com/sqlancer/sqlancer/issues/400. --- src/sqlancer/postgres/gen/PostgresSetGenerator.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index bfc0bbc04..f600f1f04 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -55,8 +55,6 @@ private enum ConfigurationOption { (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), VACUUM_MULTIXACT_FREEZE_MIN_AGE("vacuum_multixact_freeze_min_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 1000000000)), - VACUUM_CLEANUP_INDEX_SCALE_FACTOR("vacuum_cleanup_index_scale_factor", - (r) -> Randomly.fromOptions(0.0, 0.0000001, 0.00001, 0.01, 0.1, 1, 10, 100, 100000, 10000000000.0)), // TODO others GIN_FUZZY_SEARCH_LIMIT("gin_fuzzy_search_limit", (r) -> r.getInteger(0, 2147483647)), // 19.13. Version and Platform Compatibility From 739b8a64aa7f22a9147b3ab67b0a9a8b6277f2f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Nov 2021 09:12:20 +0000 Subject: [PATCH 037/856] Bump spotbugs-maven-plugin from 4.4.2.2 to 4.5.0.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.4.2.2 to 4.5.0.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.4.2.2...spotbugs-maven-plugin-4.5.0.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e75164b9a..9a998533e 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.4.2.2 + 4.5.0.0 spotbugs From 2ed71c3f432000cf7942263db6e5f934fc50dc92 Mon Sep 17 00:00:00 2001 From: entropaos Date: Wed, 8 Dec 2021 04:48:13 -0800 Subject: [PATCH 038/856] Fix union operator --- src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java | 2 +- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java index e3e522de1..b2594522a 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryRangeOperation.java @@ -10,7 +10,7 @@ public class PostgresBinaryRangeOperation extends BinaryNode private final String op; public enum PostgresBinaryRangeOperator implements Operator { - UNION("*"), INTERSECTION("*"), DIFFERENCE("-"); + UNION("+"), INTERSECTION("*"), DIFFERENCE("-"); private final String textRepresentation; diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 62a8a811b..771fc8651 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -114,6 +114,7 @@ public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); } public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { From 3f29b049d784cc3b6491aa7f9f767672f876dd83 Mon Sep 17 00:00:00 2001 From: bashaojing Date: Wed, 15 Dec 2021 11:43:01 +0800 Subject: [PATCH 039/856] support oceanbase --- .travis.yml | 6 + src/sqlancer/Main.java | 2 + src/sqlancer/oceanbase/OceanBaseErrors.java | 48 ++ .../OceanBaseExpectedValueVisitor.java | 155 +++++ .../oceanbase/OceanBaseGlobalState.java | 20 + .../oceanbase/OceanBaseHintGenerator.java | 133 ++++ src/sqlancer/oceanbase/OceanBaseOptions.java | 60 ++ src/sqlancer/oceanbase/OceanBaseProvider.java | 161 +++++ src/sqlancer/oceanbase/OceanBaseSchema.java | 291 ++++++++ .../oceanbase/OceanBaseToStringVisitor.java | 306 +++++++++ src/sqlancer/oceanbase/OceanBaseVisitor.java | 110 +++ .../oceanbase/ast/OceanBaseAggregate.java | 26 + .../OceanBaseBinaryComparisonOperation.java | 122 ++++ .../ast/OceanBaseBinaryLogicalOperation.java | 110 +++ .../oceanbase/ast/OceanBaseCastOperation.java | 36 + .../oceanbase/ast/OceanBaseColumnName.java | 17 + .../ast/OceanBaseColumnReference.java | 41 ++ .../ast/OceanBaseComputableFunction.java | 270 ++++++++ .../oceanbase/ast/OceanBaseConstant.java | 628 ++++++++++++++++++ .../oceanbase/ast/OceanBaseExists.java | 30 + .../oceanbase/ast/OceanBaseExpression.java | 9 + .../oceanbase/ast/OceanBaseInOperation.java | 58 ++ src/sqlancer/oceanbase/ast/OceanBaseJoin.java | 10 + .../oceanbase/ast/OceanBaseOrderByTerm.java | 36 + .../oceanbase/ast/OceanBaseSelect.java | 58 ++ .../ast/OceanBaseStringExpression.java | 22 + .../ast/OceanBaseTableReference.java | 17 + src/sqlancer/oceanbase/ast/OceanBaseText.java | 27 + .../ast/OceanBaseUnaryPostfixOperation.java | 58 ++ .../ast/OceanBaseUnaryPrefixOperation.java | 91 +++ .../oceanbase/gen/OceanBaseAlterTable.java | 72 ++ .../gen/OceanBaseDeleteGenerator.java | 52 ++ .../oceanbase/gen/OceanBaseDropIndex.java | 30 + .../gen/OceanBaseExpressionGenerator.java | 212 ++++++ .../gen/OceanBaseInsertGenerator.java | 110 +++ .../gen/OceanBaseTableGenerator.java | 282 ++++++++ .../gen/OceanBaseTruncateTableGenerator.java | 18 + .../gen/OceanBaseUpdateGenerator.java | 54 ++ .../gen/datadef/OceanBaseIndexGenerator.java | 150 +++++ .../oracle/OceanBaseNoRECOracle.java | 204 ++++++ .../OceanBasePivotedQuerySynthesisOracle.java | 159 +++++ .../oceanbase/oracle/OceanBaseTLPBase.java | 62 ++ .../oracle/OceanBaseTLPWhereOracle.java | 44 ++ test/sqlancer/dbms/TestOceanBaseNoREC.java | 29 + test/sqlancer/dbms/TestOceanBasePQS.java | 29 + test/sqlancer/dbms/TestOceanBaseTLP.java | 29 + 46 files changed, 4494 insertions(+) create mode 100644 src/sqlancer/oceanbase/OceanBaseErrors.java create mode 100644 src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java create mode 100644 src/sqlancer/oceanbase/OceanBaseGlobalState.java create mode 100644 src/sqlancer/oceanbase/OceanBaseHintGenerator.java create mode 100644 src/sqlancer/oceanbase/OceanBaseOptions.java create mode 100644 src/sqlancer/oceanbase/OceanBaseProvider.java create mode 100644 src/sqlancer/oceanbase/OceanBaseSchema.java create mode 100644 src/sqlancer/oceanbase/OceanBaseToStringVisitor.java create mode 100644 src/sqlancer/oceanbase/OceanBaseVisitor.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseAggregate.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseColumnName.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseConstant.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseExists.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseExpression.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseInOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseJoin.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseSelect.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseTableReference.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseText.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java create mode 100644 src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java create mode 100644 src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java create mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java create mode 100644 src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java create mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java create mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java create mode 100644 test/sqlancer/dbms/TestOceanBaseNoREC.java create mode 100644 test/sqlancer/dbms/TestOceanBasePQS.java create mode 100644 test/sqlancer/dbms/TestOceanBaseTLP.java diff --git a/.travis.yml b/.travis.yml index 318ea6fa7..6e5594948 100644 --- a/.travis.yml +++ b/.travis.yml @@ -144,3 +144,9 @@ matrix: - sleep 5 script: - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test + - name: OceanBase + jdk : openjdk8 + script: + - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBaseTLP test + - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBasePQS test + - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBaseNoREC test diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 20024e437..1ab9e29d9 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -37,6 +37,7 @@ import sqlancer.postgres.PostgresProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; +import sqlancer.oceanbase.OceanBaseProvider; public final class Main { @@ -560,6 +561,7 @@ private boolean run(MainOptions options, ExecutorService execService, providers.add(new MongoDBProvider()); providers.add(new CosmosProvider()); providers.add(new ArangoDBProvider()); + providers.add(new OceanBaseProvider()); return providers; } diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java new file mode 100644 index 000000000..392bd7c5e --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -0,0 +1,48 @@ +package sqlancer.oceanbase; + +import sqlancer.common.query.ExpectedErrors; + +public final class OceanBaseErrors { + + private OceanBaseErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) + errors.add("is not valid for CHARACTER SET"); + errors.add("The observer or zone is not the master"); + errors.add("Incorrect integer value"); + errors.add("Truncated incorrect DOUBLE value"); + errors.add("Invalid numeric"); + errors.add("Data truncated for argument"); + } + public static void addInsertErrors(ExpectedErrors errors) { + errors.add("Duplicate entry"); + errors.add("cannot be null"); + errors.add("doesn't have a default value"); + errors.add("Out of range value"); + errors.add("Incorrect double value"); + errors.add("Incorrect float value"); + errors.add("Incorrect int value"); + errors.add("Incorrect tinyint value"); + errors.add("Data truncation"); + errors.add("Bad Number"); + errors.add("The value specified for generated column"); // TODO: do not insert data into generated columns + errors.add("incorrect utf8 value"); + errors.add("Data truncation: %s value is out of range in '%s'"); + errors.add("Incorrect smallint value"); + errors.add("Incorrect bigint value"); + errors.add("Incorrect decimal value"); + errors.add("error parsing regexp"); + errors.add("The observer or zone is not the master"); + errors.add("Incorrect integer value"); + errors.add("Truncated incorrect DOUBLE value"); + errors.add("Data truncated for argument"); + errors.add("Invalid numeric"); + + + if (true) { + errors.add("Miss column"); + } + } +} diff --git a/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java new file mode 100644 index 000000000..2c03436d1 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java @@ -0,0 +1,155 @@ +package sqlancer.oceanbase; + +import sqlancer.IgnoreMeException; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseColumnName; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; + +public class OceanBaseExpectedValueVisitor implements OceanBaseVisitor { + + private final StringBuilder sb = new StringBuilder(); + private int nrTabs; + + private void print(OceanBaseExpression expr) { + OceanBaseToStringVisitor v = new OceanBaseToStringVisitor(); + v.visit(expr); + for (int i = 0; i < nrTabs; i++) { + sb.append("\t"); + } + sb.append(v.get()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + @Override + public void visit(OceanBaseExpression expr) { + nrTabs++; + try { + OceanBaseVisitor.super.visit(expr); + } catch (IgnoreMeException e) { + + } + nrTabs--; + } + + @Override + public void visit(OceanBaseConstant constant) { + print(constant); + } + + @Override + public void visit(OceanBaseColumnReference column) { + print(column); + } + + @Override + public void visit(OceanBaseUnaryPostfixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(OceanBaseComputableFunction f) { + print(f); + for (OceanBaseExpression expr : f.getArguments()) { + visit(expr); + } + } + + @Override + public void visit(OceanBaseBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public String get() { + return sb.toString(); + } + + @Override + public void visit(OceanBaseSelect select) { + for (OceanBaseExpression j : select.getJoinList()) { + visit(j); + } + if (select.getWhereClause() != null) { + visit(select.getWhereClause()); + } + } + + @Override + public void visit(OceanBaseBinaryComparisonOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(OceanBaseCastOperation op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(OceanBaseInOperation op) { + print(op); + visit(op.getExpr()); + for (OceanBaseExpression right : op.getListElements()) { + visit(right); + } + } + + @Override + public void visit(OceanBaseOrderByTerm op) { + } + + @Override + public void visit(OceanBaseExists op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(OceanBaseStringExpression op) { + print(op); + } + + @Override + public void visit(OceanBaseTableReference ref) { + } + + @Override + public void visit(OceanBaseAggregate aggr) { + } + + @Override + public void visit(OceanBaseColumnName aggr) { + } + + @Override + public void visit(OceanBaseText func) { + } + + @Override + public void visit(OceanBaseUnaryPrefixOperation op){ + print(op); + visit(op.getExpr()); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseGlobalState.java b/src/sqlancer/oceanbase/OceanBaseGlobalState.java new file mode 100644 index 000000000..b8baf08fb --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseGlobalState.java @@ -0,0 +1,20 @@ + +package sqlancer.oceanbase; + +import java.sql.SQLException; + +import sqlancer.SQLGlobalState; +import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; + +public class OceanBaseGlobalState extends SQLGlobalState { + + @Override + protected OceanBaseSchema readSchema() throws SQLException { + return OceanBaseSchema.fromConnection(getConnection(), getDatabaseName()); + } + + public boolean usesPQS() { + return getDbmsSpecificOptions().oracles.stream().anyMatch(o -> o == OceanBaseOracleFactory.PQS); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java new file mode 100644 index 000000000..83accab72 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java @@ -0,0 +1,133 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; + +import java.util.List; +import java.util.stream.Collectors; + +public class OceanBaseHintGenerator { + private OceanBaseSelect select; + private List tables; + private final StringBuilder sb = new StringBuilder(); + private Randomly r = new Randomly(); + + enum IndexHint { + MERGE_JOIN, + INL_JOIN, + INL_HASH_JOIN, + INL_MERGE_JOIN, + HASH_JOIN, + HASH_AGG, + STREAM_AGG, + USE_INDEX, + IGNORE_INDEX, + AGG_TO_COP, + USE_INDEX_MERGE, + NO_INDEX_MERGE, + LEADING, + PredDeduce, + PDML, + USE_TOJA; + } + + public OceanBaseHintGenerator(OceanBaseSelect select, List tables) { + this.select = select; + this.tables = tables; + } + + public static void generateHints(OceanBaseSelect select, List tables) { + new OceanBaseHintGenerator(select, tables).generate(); + + } + + private void generate() { + OceanBaseTable table = Randomly.fromList(tables); + switch (Randomly.fromOptions(IndexHint.values())) { + case PDML: + sb.append(" parallel(" + r.getInteger(0, 10) + "),enable_parallel_dml "); + break; + case PredDeduce: + sb.append("no_pred_deduce"); + break; + case MERGE_JOIN: + tablesHint("USE_MERGE "); + break; + case INL_JOIN: + tablesHint("USE_NL "); + break; + case LEADING: + tablesHint(" LEADING "); + break; + case INL_HASH_JOIN: + tablesHint("USE_HASH "); + break; + case INL_MERGE_JOIN: + tablesHint("USE_BNL "); + break; + case HASH_JOIN: + sb.append(" parallel(1) "); + break; + case HASH_AGG: + sb.append("USE_HASH_AGGREGATION "); + break; + case STREAM_AGG: + sb.append("USE_NL_MATERIALIZATION "); + break; + case USE_INDEX: + indexesHint("INDEX_HINT "); + break; + case IGNORE_INDEX: + sb.append("TOPK (50 50) "); + break; + case AGG_TO_COP: + sb.append("USE_LATE_MATERIALIZATION "); + break; + case USE_INDEX_MERGE: + sb.append("ORDERED "); + break; + case NO_INDEX_MERGE: + tablesHint("NO_MERGE "); + break; + case USE_TOJA: + sb.append("no_rewrite " ); + break; + default: + throw new AssertionError(); + } + + select.setHint(new OceanBaseStringExpression(sb.toString(),new OceanBaseConstant.OceanBaseTextConstant(sb.toString()))); + } + + private void indexesHint(String string) { + sb.append(string); + sb.append("("); + OceanBaseTable table = Randomly.fromList(tables); + List allIndexes = table.getIndexes(); + if (allIndexes.isEmpty()) { + throw new IgnoreMeException(); + } + List indexSubset = Randomly.nonEmptySubset(allIndexes); + sb.append(table.getName()); + sb.append(", "); + sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); + sb.append(")"); + } + + private void tablesHint(String string) { + sb.append(string); + sb.append("("); + appendTables(); + sb.append(")"); + } + + private void appendTables() { + List tableSubset = Randomly.nonEmptySubset(tables); + sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + } +} diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java new file mode 100644 index 000000000..fb529e375 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -0,0 +1,60 @@ +package sqlancer.oceanbase; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; +import sqlancer.oceanbase.oracle.OceanBasePivotedQuerySynthesisOracle; +import sqlancer.oceanbase.oracle.OceanBaseTLPWhereOracle; +import sqlancer.oceanbase.oracle.OceanBaseNoRECOracle; + +@Parameters(separators = "=", commandDescription = "OceanBase (default port: " + OceanBaseOptions.DEFAULT_PORT + + ", default host: " + OceanBaseOptions.DEFAULT_HOST) +public class OceanBaseOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 2883; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(OceanBaseOracleFactory.TLP_WHERE); + + public enum OceanBaseOracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + return new OceanBaseTLPWhereOracle(globalState); + } + }, + NoREC { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + return new OceanBaseNoRECOracle(globalState); + } + }, + PQS { + + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + return new OceanBasePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + } + } + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseProvider.java b/src/sqlancer/oceanbase/OceanBaseProvider.java new file mode 100644 index 000000000..5b57ee3c1 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseProvider.java @@ -0,0 +1,161 @@ +package sqlancer.oceanbase; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import sqlancer.AbstractAction; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.oceanbase.gen.OceanBaseAlterTable; +import sqlancer.oceanbase.gen.OceanBaseDeleteGenerator; +import sqlancer.oceanbase.gen.OceanBaseDropIndex; +import sqlancer.oceanbase.gen.OceanBaseInsertGenerator; +import sqlancer.oceanbase.gen.OceanBaseTableGenerator; +import sqlancer.oceanbase.gen.OceanBaseTruncateTableGenerator; +import sqlancer.oceanbase.gen.OceanBaseUpdateGenerator; +import sqlancer.oceanbase.gen.datadef.OceanBaseIndexGenerator; + +public class OceanBaseProvider extends SQLProviderAdapter { + + public OceanBaseProvider() { + super(OceanBaseGlobalState.class, OceanBaseOptions.class); + } + + enum Action implements AbstractAction { + SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), + INSERT(OceanBaseInsertGenerator::insertRow), + CREATE_INDEX(OceanBaseIndexGenerator::create), + ALTER_TABLE(OceanBaseAlterTable::create), + TRUNCATE_TABLE(OceanBaseTruncateTableGenerator::generate), + SELECT_INFO((g) -> new SQLQueryAdapter( + "select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + g.getDatabaseName() + + "'")), + CREATE_TABLE((g) -> { + String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); + + return OceanBaseTableGenerator.generate(g, tableName); + }), + DELETE(OceanBaseDeleteGenerator::delete), + UPDATE(OceanBaseUpdateGenerator::getQuery), + DROP_INDEX(OceanBaseDropIndex::generate); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(OceanBaseGlobalState globalState) throws Exception { + return sqlQueryProvider.getQuery(globalState); + } + } + + private static int mapActions(OceanBaseGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed = 0; + switch (a) { + case DROP_INDEX: + nrPerformed = r.getInteger(0, 2); + break; + case SHOW_TABLES: + nrPerformed = r.getInteger(0, 1); + break; + case CREATE_TABLE: + nrPerformed = r.getInteger(0, 1); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 5); + break; + case ALTER_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case TRUNCATE_TABLE: + nrPerformed = r.getInteger(0, 2); + break; + case SELECT_INFO: + nrPerformed = r.getInteger(0, 10); + break; + case DELETE: + nrPerformed = r.getInteger(0, 10); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + } + + @Override + public void generateDatabase(OceanBaseGlobalState globalState) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = OceanBaseTableGenerator.generate(globalState, tableName); + globalState.executeStatement(createTable); + } + + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + OceanBaseProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exception,SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = OceanBaseOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = OceanBaseOptions.DEFAULT_PORT; + } + if(username.endsWith("sys")||username.equals("root")) + { + throw new Exception("please don't use sys tenant to test! Firstly create tenant then test"); + } + String databaseName = globalState.getDatabaseName(); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); + + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "oceanbase"; + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseSchema.java b/src/sqlancer/oceanbase/OceanBaseSchema.java new file mode 100644 index 000000000..a16358110 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseSchema.java @@ -0,0 +1,291 @@ +package sqlancer.oceanbase; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.IgnoreMeException; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseConstant; + +public class OceanBaseSchema extends AbstractSchema { + + private static final int NR_SCHEMA_READ_TRIES = 10; + + public enum OceanBaseDataType { + INT, VARCHAR, FLOAT, DOUBLE, DECIMAL; + + public static OceanBaseDataType getRandom(OceanBaseGlobalState globalState) { + if (globalState.usesPQS()) { + return Randomly.fromOptions(OceanBaseDataType.INT, OceanBaseDataType.VARCHAR); + } else { + return Randomly.fromOptions(values()); + } + } + + public boolean isNumeric() { + switch (this) { + case INT: + case DOUBLE: + case FLOAT: + case DECIMAL: + return true; + case VARCHAR: + return false; + default: + throw new AssertionError(this); + } + } + + } + + public static class OceanBaseColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isZeroFill; + private final int precision; + public boolean isPartioned; + + public enum CollateSequence { + NOCASE, RTRIM, BINARY; + + public static CollateSequence random() { + return Randomly.fromOptions(values()); + + } + } + + public OceanBaseColumn(String name, OceanBaseDataType columnType, boolean isPrimaryKey, int precision, boolean isZeroFill) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.precision = precision; + this.isPartioned = true; + this.isZeroFill = isZeroFill; + } + + public int getPrecision() { + return precision; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isZeroFill() { + return isZeroFill; + } + + } + + public static class OceanBaseTables extends AbstractTables { + + public OceanBaseTables(List tables) { + super(tables); + } + + public OceanBaseRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", columnNamesAsString( + c -> c.getType()==OceanBaseDataType.FLOAT || c.isZeroFill() ? ("concat(" + c.getTable().getName() + "." + c.getName() + ",'')" + " AS " + c.getTable().getName() + c.getName()) : (c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName())), + tableNamesAsString()); + //cast float and zerofill as varchar + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet randomRowValues = s.executeQuery(randomRow); + if (!randomRowValues.next()) { + throw new IgnoreMeException(); + } + for (int i = 0; i < getColumns().size(); i++) { + OceanBaseColumn column = getColumns().get(i); + Object value; + int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + OceanBaseConstant constant; + if (randomRowValues.getString(columnIndex) == null) { + if(column.isZeroFill()) + constant= OceanBaseConstant.createStringConstant("null"); + else + constant = OceanBaseConstant.createNullConstant(); + + } else { + switch (column.getType()) { + case INT: + //cast zerofill as varchar + if (column.isZeroFill()){ + value = randomRowValues.getString(columnIndex); + constant = OceanBaseConstant.createStringConstant((String) value); + }else{ + value = randomRowValues.getLong(columnIndex); + constant = OceanBaseConstant.createIntConstant((long) value); + } + break; + case VARCHAR: + value = randomRowValues.getString(columnIndex); + constant = OceanBaseConstant.createStringConstant((String) value); + break; + default: + throw new AssertionError(column.getType()); + } + } + values.put(column, constant); + } + assert !randomRowValues.next(); + return new OceanBaseRowValue(this, values); + } + + } + + } + + private static OceanBaseDataType getColumnType(String typeString) { + switch (typeString) { + case "tinyint": + case "smallint": + case "mediumint": + case "int": + case "bigint": + return OceanBaseDataType.INT; + case "varchar": + case "tinytext": + case "mediumtext": + case "text": + case "longtext": + return OceanBaseDataType.VARCHAR; + case "double": + return OceanBaseDataType.DOUBLE; + case "float": + return OceanBaseDataType.FLOAT; + case "decimal": + return OceanBaseDataType.DECIMAL; + default: + throw new AssertionError(typeString); + } + } + + public static class OceanBaseRowValue extends AbstractRowValue { + + OceanBaseRowValue(OceanBaseTables tables, Map values) { + super(tables, values); + } + + } + + public static class OceanBaseTable extends AbstractRelationalTable { + + public OceanBaseTable(String tableName, List columns, List indexes) { + super(tableName, columns, indexes, false); + } + + public boolean hasPrimaryKey() { + return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); + } + + } + + public static final class OceanBaseIndex extends TableIndex { + + private OceanBaseIndex(String indexName) { + super(indexName); + } + + public static OceanBaseIndex create(String indexName) { + return new OceanBaseIndex(indexName); + } + + @Override + public String getIndexName() { + return super.getIndexName(); + + } + + } + + public static OceanBaseSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + Exception ex = null; + for (int i = 0; i < NR_SCHEMA_READ_TRIES; i++) { + try { + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + "select TABLE_NAME from information_schema.TABLES where table_schema = '" + + databaseName + "';")) { + while (rs.next()) { + String tableName = rs.getString("TABLE_NAME"); + List databaseColumns = getTableColumns(con, tableName, databaseName); + List indexes = getIndexes(con, tableName, databaseName); + OceanBaseTable t = new OceanBaseTable(tableName, databaseColumns, indexes); + for (OceanBaseColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + return new OceanBaseSchema(databaseTables); + } catch (SQLIntegrityConstraintViolationException e) { + ex = e; + } + } + throw new AssertionError(ex); + } + + private static List getIndexes(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME='%s';", + databaseName, tableName))) { + while (rs.next()) { + String indexName = rs.getString("INDEX_NAME"); + if(!indexName.equals("PRIMARY")) + indexes.add(OceanBaseIndex.create(indexName)); + } + } + } + return indexes; + } + + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("select * from information_schema.columns where table_schema = '" + + databaseName + "' AND TABLE_NAME='" + tableName + "'")) { + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + String dataType = rs.getString("DATA_TYPE"); + int precision = rs.getInt("NUMERIC_PRECISION"); + boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); + boolean isZeroFill = rs.getString("COLUMN_TYPE").contains("zerofill"); + + OceanBaseColumn c = new OceanBaseColumn(columnName, getColumnType(dataType), isPrimaryKey, precision, isZeroFill); + columns.add(c); + } + } + } + return columns; + } + + public OceanBaseSchema(List databaseTables) { + super(databaseTables); + } + + public OceanBaseTables getRandomTableNonEmptyTables() { + return new OceanBaseTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java new file mode 100644 index 000000000..053731852 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java @@ -0,0 +1,306 @@ +package sqlancer.oceanbase; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseColumnName; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; + +public class OceanBaseToStringVisitor extends ToStringVisitor implements OceanBaseVisitor { + + int ref; + private Randomly r = new Randomly(); + + @Override + public void visitSpecific(OceanBaseExpression expr) { + OceanBaseVisitor.super.visit(expr); + } + + @Override + public void visit(OceanBaseSelect s) { + sb.append("SELECT "); + if (s.getHint() != null) { + sb.append("/*+ "); + visit(s.getHint(),0); + sb.append(" */ "); + } + + switch (s.getFromOptions()) { + case DISTINCT: + sb.append("DISTINCT "); + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + sb.append(s.getModifiers().stream().collect(Collectors.joining(" "))); + if (s.getModifiers().size() > 0) { + sb.append(" "); + } + if (s.getFetchColumns() == null) { + sb.append("*"); + } else { + for (int i = 0; i < s.getFetchColumns().size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(s.getFetchColumns().get(i)); + } + } + sb.append(" FROM "); + for (int i = 0; i < s.getFromList().size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(s.getFromList().get(i)); + } + for (OceanBaseExpression j : s.getJoinList()) { + visit(j); + } + + if (s.getWhereClause() != null) { + OceanBaseExpression whereClause = s.getWhereClause(); + sb.append(" WHERE "); + visit(whereClause); + } + if (s.getGroupByExpressions() != null && s.getGroupByExpressions().size() > 0) { + sb.append(" "); + sb.append("GROUP BY "); + List groupBys = s.getGroupByExpressions(); + for (int i = 0; i < groupBys.size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(groupBys.get(i)); + } + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + } + if (!s.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + List orderBys = s.getOrderByExpressions(); + for (int i = 0; i < orderBys.size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(s.getOrderByExpressions().get(i)); + } + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(OceanBaseConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(OceanBaseColumnReference column) { + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) + {sb.append("concat(");} + sb.append(column.getColumn().getFullQualifiedName()); + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) + {sb.append(",'')");} + if (column.getRef()){ + sb.append(" AS "); + sb.append(column.getColumn().getTable().getName()); + sb.append(column.getColumn().getName()); + } + } + + @Override + public void visit(OceanBaseUnaryPostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" IS "); + if (op.isNegated()) { + sb.append("NOT "); + } + switch (op.getOperator()) { + case IS_FALSE: + sb.append("FALSE"); + break; + case IS_NULL: + if (Randomly.getBoolean()) { + sb.append("UNKNOWN"); + } else { + sb.append("NULL"); + } + break; + case IS_TRUE: + sb.append("TRUE"); + break; + default: + throw new AssertionError(op); + } + } + + @Override + public void visit(OceanBaseComputableFunction f) { + sb.append(f.getFunction().getName()); + sb.append("("); + for (int i = 0; i < f.getArguments().length; i++) { + if (i != 0) { + sb.append(", "); + } + visit(f.getArguments()[i]); + } + sb.append(")"); + } + + @Override + public void visit(OceanBaseBinaryLogicalOperation op) { + sb.append("("); + visit(op.getLeft()); + sb.append(")"); + sb.append(" "); + sb.append(op.getTextRepresentation()); + sb.append(" "); + sb.append("("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseBinaryComparisonOperation op) { + sb.append("("); + visit(op.getLeft()); + sb.append(") "); + sb.append(op.getOp().getTextRepresentation()); + sb.append(" ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseCastOperation op) { + sb.append("CAST("); + visit(op.getExpr()); + sb.append(" AS "); + sb.append(op.getType()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN "); + sb.append("("); + for (int i = 0; i < op.getListElements().size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(op.getListElements().get(i)); + } + sb.append(")"); + } + + @Override + public void visit(OceanBaseOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder() == OceanBaseOrder.ASC ? "ASC" : "DESC"); + } + + @Override + public void visit(OceanBaseExists op) { + sb.append(" EXISTS ("); + visit(op.getExpr()); + sb.append(")"); + } + + @Override + public void visit(OceanBaseStringExpression op) { + if(op.getStr().contains("SELECT")){ + sb.append(op.getStr()); + } + else{ + String str = op.getStr(); + if(str.length() > 0){ + sb.append(r.getInteger(0, 100000)); + } + else{ + sb.append(r.getInteger(0, 1000000)); + } + } + } + + public void visit(OceanBaseStringExpression op,int type) { + sb.append(op.getStr()); + } + + @Override + public void visit(OceanBaseTableReference ref) { + sb.append(ref.getTable().getName()); + } + + @Override + public void visit(OceanBaseAggregate aggr) { + sb.append(aggr.getAggr()); + sb.append("("); + visit(aggr.getExpr()); + sb.append(")"); + } + @Override + public void visit(OceanBaseColumnName c) { + sb.append(c.getColumn().getName()); + } + @Override + public void visit(OceanBaseText func) { + visit(func.getExpr()); + sb.append(func.getText()); + } + + @Override + public void visit(OceanBaseUnaryPrefixOperation op) { + sb.append("("); + sb.append(op.getOp().getTextRepresentation()); + sb.append(" "); + visit(op.getExpr()); + sb.append(")"); + } + +} diff --git a/src/sqlancer/oceanbase/OceanBaseVisitor.java b/src/sqlancer/oceanbase/OceanBaseVisitor.java new file mode 100644 index 000000000..5babfd7df --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseVisitor.java @@ -0,0 +1,110 @@ +package sqlancer.oceanbase; + +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseColumnName; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; + +public interface OceanBaseVisitor { + + void visit(OceanBaseTableReference ref); + + void visit(OceanBaseConstant constant); + + void visit(OceanBaseColumnReference column); + + void visit(OceanBaseUnaryPostfixOperation column); + + void visit(OceanBaseComputableFunction f); + + void visit(OceanBaseBinaryLogicalOperation op); + + void visit(OceanBaseSelect select); + + void visit(OceanBaseBinaryComparisonOperation op); + + void visit(OceanBaseCastOperation op); + + void visit(OceanBaseInOperation op); + + void visit(OceanBaseOrderByTerm op); + + void visit(OceanBaseExists op); + + void visit(OceanBaseStringExpression op); + + void visit(OceanBaseAggregate aggr); + + void visit(OceanBaseColumnName c); + + void visit(OceanBaseText fun); + + void visit(OceanBaseUnaryPrefixOperation op); + + default void visit(OceanBaseExpression expr) { + if (expr instanceof OceanBaseConstant) { + visit((OceanBaseConstant) expr); + } else if (expr instanceof OceanBaseColumnReference) { + visit((OceanBaseColumnReference) expr); + } else if (expr instanceof OceanBaseUnaryPostfixOperation) { + visit((OceanBaseUnaryPostfixOperation) expr); + } else if (expr instanceof OceanBaseComputableFunction) { + visit((OceanBaseComputableFunction) expr); + } else if (expr instanceof OceanBaseBinaryLogicalOperation) { + visit((OceanBaseBinaryLogicalOperation) expr); + } else if (expr instanceof OceanBaseSelect) { + visit((OceanBaseSelect) expr); + } else if (expr instanceof OceanBaseBinaryComparisonOperation) { + visit((OceanBaseBinaryComparisonOperation) expr); + } else if (expr instanceof OceanBaseCastOperation) { + visit((OceanBaseCastOperation) expr); + } else if (expr instanceof OceanBaseInOperation) { + visit((OceanBaseInOperation) expr); + } else if (expr instanceof OceanBaseOrderByTerm) { + visit((OceanBaseOrderByTerm) expr); + } else if (expr instanceof OceanBaseExists) { + visit((OceanBaseExists) expr); + } else if (expr instanceof OceanBaseStringExpression) { + visit((OceanBaseStringExpression) expr); + } else if (expr instanceof OceanBaseTableReference) { + visit((OceanBaseTableReference) expr); + } else if (expr instanceof OceanBaseAggregate) { + visit((OceanBaseAggregate) expr); + } else if (expr instanceof OceanBaseColumnName) { + visit((OceanBaseColumnName) expr); + } else if (expr instanceof OceanBaseText) { + visit((OceanBaseText) expr); + } else if (expr instanceof OceanBaseUnaryPrefixOperation) { + visit((OceanBaseUnaryPrefixOperation) expr); + } else { + throw new AssertionError(expr); + } + } + + static String asString(OceanBaseExpression expr) { + OceanBaseToStringVisitor visitor = new OceanBaseToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + static String asExpectedValues(OceanBaseExpression expr) { + OceanBaseExpectedValueVisitor visitor = new OceanBaseExpectedValueVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java new file mode 100644 index 000000000..ddf50b745 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java @@ -0,0 +1,26 @@ +package sqlancer.oceanbase.ast; + + +public class OceanBaseAggregate implements OceanBaseExpression { + + private OceanBaseExpression expr; + private OceanBaseAggregateFunction aggr; + + public OceanBaseAggregate(OceanBaseExpression expr, OceanBaseAggregateFunction aggr) { + this.expr = expr; + this.aggr = aggr; + } + + public enum OceanBaseAggregateFunction { + COUNT + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public OceanBaseAggregateFunction getAggr() { + return aggr; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java new file mode 100644 index 000000000..3fe3bd32b --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java @@ -0,0 +1,122 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.LikeImplementationHelper; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; + +public class OceanBaseBinaryComparisonOperation implements OceanBaseExpression { + + public enum BinaryComparisonOperator { + EQUALS("=") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + return leftVal.isEquals(rightVal); + } + }, + NOT_EQUALS("!=") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant isEquals = leftVal.isEquals(rightVal); + if (isEquals.getType() == OceanBaseDataType.INT) { + return OceanBaseConstant.createIntConstant(1 - isEquals.getInt()); + } + return isEquals; + } + }, + LESS("<") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + return leftVal.isLessThan(rightVal); + } + }, + LESS_EQUALS("<=") { + + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant lessThan = leftVal.isLessThan(rightVal); + if (lessThan == null) { + return null; + } + if (lessThan.getType() == OceanBaseDataType.INT && lessThan.getInt() == 0) { + return leftVal.isEquals(rightVal); + } else { + return lessThan; + } + } + }, + GREATER(">") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant equals = leftVal.isEquals(rightVal); + if (equals.getType() == OceanBaseDataType.INT && equals.getInt() == 1) { + return OceanBaseConstant.createFalse(); + } else { + OceanBaseConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + return OceanBaseUnaryPrefixOperator.NOT.applyNotNull(applyLess); + } + } + }, + GREATER_EQUALS(">=") { + @Override + public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal) { + OceanBaseConstant equals = leftVal.isEquals(rightVal); + if (equals.getType() == OceanBaseDataType.INT && equals.getInt() == 1) { + return OceanBaseConstant.createTrue(); + } else { + OceanBaseConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + return OceanBaseUnaryPrefixOperator.NOT.applyNotNull(applyLess); + } + } + }; + private final String textRepresentation; + + public String getTextRepresentation() { + return textRepresentation; + } + + BinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public abstract OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseConstant rightVal); + + public static BinaryComparisonOperator getRandom() { + return Randomly.fromOptions(BinaryComparisonOperator.values()); + } + } + + private final OceanBaseExpression left; + private final OceanBaseExpression right; + private final BinaryComparisonOperator op; + + public OceanBaseBinaryComparisonOperation(OceanBaseExpression left, OceanBaseExpression right, BinaryComparisonOperator op) { + this.left = left; + this.right = right; + this.op = op; + } + + public OceanBaseExpression getLeft() { + return left; + } + + public BinaryComparisonOperator getOp() { + return op; + } + + public OceanBaseExpression getRight() { + return right; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return op.getExpectedValue(left.getExpectedValue(), right.getExpectedValue()); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java new file mode 100644 index 000000000..0cb9ded11 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java @@ -0,0 +1,110 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.Randomly; + +public class OceanBaseBinaryLogicalOperation implements OceanBaseExpression { + + private final OceanBaseExpression left; + private final OceanBaseExpression right; + private final OceanBaseBinaryLogicalOperator op; + private final String textRepresentation; + + public enum OceanBaseBinaryLogicalOperator { + AND("AND", "&&") { + @Override + public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) { + if (left.isNull() && right.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (left.isNull()) { + if (right.asBooleanNotNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createFalse(); + } + } else if (right.isNull()) { + if (left.asBooleanNotNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createFalse(); + } + } else { + return OceanBaseConstant.createBoolean(left.asBooleanNotNull() && right.asBooleanNotNull()); + } + } + }, + OR("OR", "||") { + @Override + public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) { + if (!left.isNull() && left.asBooleanNotNull()) { + return OceanBaseConstant.createTrue(); + } else if (!right.isNull() && right.asBooleanNotNull()) { + return OceanBaseConstant.createTrue(); + } else if (left.isNull() || right.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createFalse(); + } + } + }, + XOR("XOR") { + @Override + public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) { + if (left.isNull() || right.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + boolean xorVal = left.asBooleanNotNull() ^ right.asBooleanNotNull(); + return OceanBaseConstant.createBoolean(xorVal); + } + }; + + private final String[] textRepresentations; + + OceanBaseBinaryLogicalOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + + public abstract OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right); + + public static OceanBaseBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public OceanBaseBinaryLogicalOperation(OceanBaseExpression left, OceanBaseExpression right, OceanBaseBinaryLogicalOperator op) { + this.left = left; + this.right = right; + this.op = op; + this.textRepresentation = op.getTextRepresentation(); + } + + public OceanBaseExpression getLeft() { + return left; + } + + public OceanBaseBinaryLogicalOperator getOp() { + return op; + } + + public OceanBaseExpression getRight() { + return right; + } + + public String getTextRepresentation() { + return textRepresentation; + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant leftExpected = left.getExpectedValue(); + OceanBaseConstant rightExpected = right.getExpectedValue(); + if (left.getExpectedValue() == null || right.getExpectedValue() == null) { + return null; + } + return op.apply(leftExpected, rightExpected); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java new file mode 100644 index 000000000..3b5ade9a3 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseCastOperation.java @@ -0,0 +1,36 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseCastOperation implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final CastType type; + + public enum CastType { + SIGNED, UNSIGNED; + + public static CastType getRandom() { + return SIGNED; + // return Randomly.fromOptions(CastType.values()); + } + + } + + public OceanBaseCastOperation(OceanBaseExpression expr, CastType type) { + this.expr = expr; + this.type = type; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public CastType getType() { + return type; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return expr.getExpectedValue().castAs(type); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java new file mode 100644 index 000000000..c6ee0f350 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java @@ -0,0 +1,17 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.oceanbase.OceanBaseSchema; + +public class OceanBaseColumnName implements OceanBaseExpression { + + private OceanBaseSchema.OceanBaseColumn column; + + public OceanBaseColumnName(OceanBaseSchema.OceanBaseColumn column) { + this.column = column; + } + + public OceanBaseSchema.OceanBaseColumn getColumn() { + return column; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java new file mode 100644 index 000000000..64f903267 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java @@ -0,0 +1,41 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; + +public class OceanBaseColumnReference implements OceanBaseExpression { + + private final OceanBaseColumn column; + private final OceanBaseConstant value; + private boolean isRef = false; + + public OceanBaseColumnReference(OceanBaseColumn column, OceanBaseConstant value) { + this.column = column; + this.value = value; + } + + public static OceanBaseColumnReference create(OceanBaseColumn column, OceanBaseConstant value) { + return new OceanBaseColumnReference(column, value); + } + + public OceanBaseColumn getColumn() { + return column; + } + + public OceanBaseConstant getValue() { + return value; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return value; + } + + public OceanBaseColumnReference setRef(boolean isRef){ + this.isRef = isRef; + return this; + } + public boolean getRef(){ + return isRef; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java new file mode 100644 index 000000000..ee78dd584 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java @@ -0,0 +1,270 @@ +package sqlancer.oceanbase.ast; + +import java.util.function.BinaryOperator; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseCastOperation.CastType; + +public class OceanBaseComputableFunction implements OceanBaseExpression { + + private final OceanBaseFunction func; + private final OceanBaseExpression[] args; + + public OceanBaseComputableFunction(OceanBaseFunction func, OceanBaseExpression... args) { + this.func = func; + this.args = args; + } + + public OceanBaseFunction getFunction() { + return func; + } + + public OceanBaseExpression[] getArguments() { + return args; + } + + public enum OceanBaseFunction { + + BIT_COUNT(1, "BIT_COUNT") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + OceanBaseConstant arg = evaluatedArgs[0]; + if (arg.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + long val = arg.castAs(CastType.SIGNED).getInt(); + return OceanBaseConstant.createIntConstant(Long.bitCount(val)); + } + } + + }, + COALESCE(2, "COALESCE") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + OceanBaseConstant result = OceanBaseConstant.createNullConstant(); + for (OceanBaseConstant arg : args) { + if (!arg.isNull()) { + result = arg; + break; + } + } + return castToMostGeneralType(result, origArgs); + } + + @Override + public boolean isVariadic() { + return true; + } + + }, + IF(3, "IF") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + OceanBaseConstant cond = args[0]; + OceanBaseConstant left = args[1]; + OceanBaseConstant right = args[2]; + OceanBaseConstant result; + if (cond.isNull() || !cond.asBooleanNotNull()) { + result = right; + } else { + result = left; + } + return castToMostGeneralType(result, new OceanBaseExpression[] { origArgs[1], origArgs[2] }); + } + }, + + IFNULL(2, "IFNULL") { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + OceanBaseConstant result; + if (args[0].isNull()) { + result = args[1]; + } else { + result = args[0]; + }//args[0] and args[1] both null, if type is varchar, return null of varchar + return castToMostGeneralType(result, origArgs); + } + + }, + LEAST(2, "LEAST", true) { + + @Override + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + return aggregate(evaluatedArgs, args, (min, cur) -> cur.isLessThan(min).asBooleanNotNull() ? cur : min); + } + + }, + GREATEST(2, "GREATEST", true) { + @Override + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + return aggregate(evaluatedArgs, args, (max, cur) -> cur.isLessThan(max).asBooleanNotNull() ? max : cur); + } + }; + + private String functionName; + final int nrArgs; + private final boolean variadic; + + private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] typeExpressions, BinaryOperator op) { + boolean containsNull = Stream.of(evaluatedArgs).anyMatch(arg -> arg.isNull()); + if (containsNull) { + //IFNULL(GREATEST('aa',NULL), 0) -> '0' + // case1:c1 is float，value is NULL;select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' + // select COALESCE(GREATEST(1, concat(t1.c1)), 1) from t1;->1 + // select COALESCE(GREATEST('0', 1, concat(t1.c1)), 1) from t1;->1 + // select COALESCE(GREATEST('0', concat(t1.c1)), 1) from t1;->'1' + // select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' + // case2: c0 is decimal,value is NULL + // select IFNULL(GREATEST("iffI|2&nBJLQQ", c0, '0'), 1) from t0;->1 + // select IFNULL(GREATEST("iffI|2&nBJLQQ", NULL, '0'), 1) from t0;->'1' + OceanBaseDataType type; + boolean allVarchar = true; + for (OceanBaseExpression expr : typeExpressions) { + if (expr instanceof OceanBaseColumnReference){ + type = ((OceanBaseColumnReference) expr).getColumn().getType(); + if(type == OceanBaseDataType.FLOAT) + type = OceanBaseDataType.VARCHAR; + }else + type = expr.getExpectedValue().getType(); + if(type != null && type.isNumeric()){ + allVarchar = false; + break; + } + } + if (allVarchar) + return OceanBaseConstant.createStringConstant("null"); + else + return OceanBaseConstant.createNullConstant(); + } + OceanBaseConstant least = evaluatedArgs[1]; + /*select least(1,'H8*GPLuBjDj#Xem]W'); -> 0 + select least('1','H8*GPLuBjDj#Xem]W'); ->1 + select LEAST('000000000001', 'b', 1);->0*/ + OceanBaseDataType dataType = evaluatedArgs[0].getType(); + boolean sameDataType = true; + for (OceanBaseConstant arg : evaluatedArgs) { + if (arg.getType() != dataType){ + sameDataType = false; + break; + } + } + for (OceanBaseConstant arg : evaluatedArgs) { + OceanBaseConstant left; + OceanBaseConstant right; + if(sameDataType){ + left = least; + right = arg; + }else{ + //select GREATEST('1.47529e18', -1188315266);->1.47529e18 + if(least.getType() == OceanBaseDataType.VARCHAR) + left = least.castAsDouble(); + else + left = least; + if(arg.getType() == OceanBaseDataType.VARCHAR) + right = arg.castAsDouble(); + else + right = arg; + } + least = op.apply(right, left); + } + return least; + } + + OceanBaseFunction(int nrArgs, String functionName) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = false; + } + + OceanBaseFunction(int nrArgs, String functionName, boolean variadic) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = variadic; + } + + public int getNrArgs() { + return nrArgs; + } + + public abstract OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args); + + public static OceanBaseFunction getRandomFunction() { + return Randomly.fromOptions(values()); + } + + @Override + public String toString() { + return functionName; + } + + public boolean isVariadic() { + return variadic; + } + + public String getName() { + return functionName; + } + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant[] constants = new OceanBaseConstant[args.length]; + for (int i = 0; i < constants.length; i++) { + constants[i] = args[i].getExpectedValue(); + } + return func.apply(constants, args); + } + + public static OceanBaseConstant castToMostGeneralType(OceanBaseConstant cons, OceanBaseExpression... typeExpressions) { + OceanBaseDataType type = getMostGeneralType(typeExpressions); + if (cons.isNull()) { + if (type == OceanBaseDataType.FLOAT || type == OceanBaseDataType.VARCHAR) + return OceanBaseConstant.createStringConstant("null"); + else + return cons; + }else{ + switch (type) { + case INT: + if (cons.isInt()) { + return cons; + } else { + return OceanBaseConstant.createIntConstant(cons.castAs(CastType.SIGNED).getInt()); + } + case VARCHAR: + return OceanBaseConstant.createStringConstant(cons.castAsString()); + default: + return cons; + } + } + } + + public static OceanBaseDataType getMostGeneralType(OceanBaseExpression... expressions) { + OceanBaseDataType type = null; + for (OceanBaseExpression expr : expressions) { + OceanBaseDataType exprType; + if (expr instanceof OceanBaseColumnReference) { + exprType = ((OceanBaseColumnReference) expr).getColumn().getType(); + if(((OceanBaseColumnReference) expr).getColumn().isZeroFill()) + exprType = OceanBaseDataType.VARCHAR; + } else { + exprType = expr.getExpectedValue().getType(); + } + if (type == null) { + type = exprType; + if (exprType == OceanBaseDataType.FLOAT) + type = OceanBaseDataType.VARCHAR; + } else if (exprType == OceanBaseDataType.VARCHAR || exprType == OceanBaseDataType.FLOAT) { + type = OceanBaseDataType.VARCHAR; + } + + } + return type; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java new file mode 100644 index 000000000..d99447fb6 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -0,0 +1,628 @@ +package sqlancer.oceanbase.ast; + +import java.math.BigInteger; +import java.math.BigDecimal; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseCastOperation.CastType; + +public abstract class OceanBaseConstant implements OceanBaseExpression { + + public boolean isInt() { + return false; + } + + public boolean isNull() { + return false; + } + + public boolean isDouble() { + return false; + } + + public boolean isEmpty() { + return false; + } + + public abstract static class OceanBaseNoPQSConstant extends OceanBaseConstant { + + @Override + public boolean asBooleanNotNull() { + throw throwException(); + } + + private RuntimeException throwException() { + throw new UnsupportedOperationException("not applicable for PQS evaluation!"); + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + return null; + } + + @Override + public OceanBaseConstant castAs(CastType type) { + throw throwException(); + } + + @Override + public String castAsString() { + throw throwException(); + + } + + @Override + public OceanBaseConstant castAsDouble() { + throw throwException(); + } + + @Override + public OceanBaseDataType getType() { + throw throwException(); + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + throw throwException(); + } + + } + + public static class OceanBaseDoubleConstant extends OceanBaseNoPQSConstant { + + private final double val; + + public OceanBaseDoubleConstant(double val) { + this.val = val; + if (Double.isInfinite(val) || Double.isNaN(val)) { + throw new IgnoreMeException(); + } + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public double getDouble() { + return this.val; + } + + @Override + public long getInt() { + return new Double(val).longValue(); + } + + @Override + public boolean asBooleanNotNull() { + return Double.compare(Math.abs(val), 0.0) != 0; + } + + @Override + public OceanBaseConstant castAs(CastType type) { + if (type == CastType.SIGNED) { + long value = new Double(val).longValue(); + if (val - value >=0.5) + value = value +1; + return new OceanBaseIntConstant(value, true); + } else if (type == CastType.UNSIGNED) { + long value = new Double(val).longValue(); + if (val - value >=0.5) + value = value +1; + return new OceanBaseIntConstant(value, false); + } else { + throw new AssertionError(); + } + } + + @Override + public String castAsString() { + return String.valueOf(new BigDecimal(val));//select IFNULL(1.713591018E9, '11') -> 1713591018 + } + + @Override + public boolean isDouble() { + return true; + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal instanceof OceanBaseIntConstant){ + return OceanBaseConstant.createBoolean(val < rightVal.getInt()); + } else if (rightVal instanceof OceanBaseDoubleConstant){ + return OceanBaseConstant.createBoolean(val < rightVal.getDouble()); + } else if (rightVal instanceof OceanBaseTextConstant){ + return isLessThan(rightVal.castAsDouble()); + } else{ + throw new AssertionError(rightVal); + } + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal instanceof OceanBaseIntConstant){ + return OceanBaseConstant.createBoolean(val == rightVal.getInt()); + } else if (rightVal instanceof OceanBaseDoubleConstant){ + return OceanBaseConstant.createBoolean(val == rightVal.getDouble()); + } else if (rightVal instanceof OceanBaseTextConstant){ + return isEquals(rightVal.castAsDouble()); + } else { + throw new AssertionError(rightVal); + } + } + @Override + public OceanBaseDataType getType() { + return OceanBaseDataType.DOUBLE; + } + + } + + public static class OceanBaseTextConstant extends OceanBaseConstant { + + private final String value; + private final boolean singleQuotes; + + public OceanBaseTextConstant(String value) { + this.value = value; + singleQuotes = Randomly.getBoolean(); + + } + + private void checkIfSmallFloatingPointText() { + boolean isSmallFloatingPointText = isString() && asBooleanNotNull() + && castAs(CastType.SIGNED).getInt() == 0; + if (isSmallFloatingPointText) { + throw new IgnoreMeException(); + } + } + @Override + public boolean isNull() { + if(value.equalsIgnoreCase("NULL")) + return true; + else + return false; + } + + @Override + public boolean isEmpty(){ + //"" " " + if (value.length() ==0 ){ + return true; + }else{ + for(int i =0;i< value.length(); i++){ + String sub =value.substring(i, i+1); + if(!sub.equals(" ")) + return false; + } + return true; + } + } + + @Override + public boolean asBooleanNotNull() { + for (int i = value.length(); i >= 1; i--) { + try { + char currentChar = value.charAt(i-1); + int currentVal= Integer.valueOf(currentChar); + if (currentVal < 48||currentVal > 57) + continue; + String substring = value.substring(0, i); + Double val = Double.valueOf(substring); + return val != 0 && !Double.isNaN(val); + } catch (NumberFormatException e) { + // ignore + } + } + return false; + } + + @Override + public String getTextRepresentation() { + StringBuilder sb = new StringBuilder(); + String quotes = singleQuotes ? "'" : "\""; + sb.append(quotes); + String text = value.replace(quotes, quotes + quotes).replace("\\", "\\\\"); + sb.append(text); + sb.append(quotes); + return sb.toString(); + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + if (isNull() || rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isInt()) { + checkIfSmallFloatingPointText(); + if (asBooleanNotNull()) { + throw new IgnoreMeException(); + } + return castAs(CastType.SIGNED).isEquals(rightVal); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return castAsDouble().isEquals(rightVal); + } else if (rightVal.isString()) { + if (isEmpty() && rightVal.isEmpty()) + return OceanBaseConstant.createBoolean(true); + else + return OceanBaseConstant.createBoolean(value.equalsIgnoreCase(rightVal.getString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public String getString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public OceanBaseConstant castAs(CastType type) { + if(isNull()) + return OceanBaseConstant.createNullConstant(); + if (type == CastType.SIGNED || type == CastType.UNSIGNED) { + String value = this.value; + while (value.startsWith(" ") || value.startsWith("\t") || value.startsWith("\n")) { + if (value.startsWith("\n")) { + throw new IgnoreMeException(); + } + value = value.substring(1); + } + for (int i = value.length(); i >= 1; i--) { + try { + //select CAST("꯵z)" AS SIGNED); ->0 + char currentChar = value.charAt(i-1); + int currentVal= Integer.valueOf(currentChar); + if (currentVal < 48||currentVal > 57) + throw new IgnoreMeException(); + String substring = value.substring(0, i); + long val = Long.parseLong(substring); + return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); + } catch (NumberFormatException e) { + throw new IgnoreMeException(); + } + } + return OceanBaseConstant.createIntConstant(0, type == CastType.SIGNED ? true : false); + } else { + throw new AssertionError(); + } + } + + @Override + public OceanBaseConstant castAsDouble() { + String value = this.value; + while (value.startsWith(" ") || value.startsWith("\t") || value.startsWith("\n")) { + if (value.startsWith("\n")) { + throw new IgnoreMeException(); + } + value = value.substring(1); + } + for (int i = value.length(); i >= 1; i--) { + try { + char currentChar = value.charAt(i-1); + int currentVal= Integer.valueOf(currentChar); + if (currentVal < 48||currentVal > 57) + //ignore special char + throw new IgnoreMeException(); + String substring = value.substring(0, i); + double val = Double.parseDouble(substring); + return OceanBaseConstant.createDoubleConstant(val); + } catch (NumberFormatException e) { + throw new IgnoreMeException(); + } + } + return OceanBaseConstant.createIntConstant(0); + } + + @Override + public String castAsString() { + return value; + } + + @Override + public OceanBaseDataType getType() { + return OceanBaseDataType.VARCHAR; + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + if (isNull() || rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isInt()) { + if (asBooleanNotNull()) { + throw new IgnoreMeException(); + } + checkIfSmallFloatingPointText(); + return castAs(rightVal.isSigned() ? CastType.SIGNED : CastType.UNSIGNED).isLessThan(rightVal); + } else if (rightVal instanceof OceanBaseDoubleConstant) { + return castAsDouble().isLessThan(rightVal); + } else if (rightVal.isString()) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(rightVal); + } + } + } + + public static class OceanBaseIntConstant extends OceanBaseConstant { + + private final long value; + private final String stringRepresentation; + private final boolean isSigned; + + public OceanBaseIntConstant(long value, boolean isSigned) { + this.value = value; + this.isSigned = isSigned; + if (value == 0 && Randomly.getBoolean()) { + stringRepresentation = "FALSE"; + } else if (value == 1 && Randomly.getBoolean()) { + stringRepresentation = "TRUE"; + } else { + if (isSigned) { + stringRepresentation = String.valueOf(value); + } else { + stringRepresentation = Long.toUnsignedString(value); + } + } + } + + public OceanBaseIntConstant(long value, String stringRepresentation) { + this.value = value; + this.stringRepresentation = stringRepresentation; + isSigned = true; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public long getInt() { + return value; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public String getTextRepresentation() { + return stringRepresentation; + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + if (rightVal.isInt()) { + return OceanBaseConstant.createBoolean(new BigInteger(getStringRepr()) + .compareTo(new BigInteger(((OceanBaseIntConstant) rightVal).getStringRepr())) == 0); + } else if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isString()) { + if (rightVal.asBooleanNotNull()) { + throw new IgnoreMeException(); + } + return isEquals(rightVal.castAs(CastType.SIGNED)); + } else if (rightVal instanceof OceanBaseDoubleConstant){ + return OceanBaseConstant.createBoolean(value == rightVal.getDouble()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public OceanBaseConstant castAs(CastType type) { + if (type == CastType.SIGNED) { + return new OceanBaseIntConstant(value, true); + } else if (type == CastType.UNSIGNED) { + return new OceanBaseIntConstant(value, false); + } else { + throw new AssertionError(); + } + } + + @Override + public String castAsString() { + if (isSigned) { + return String.valueOf(value); + } else { + return Long.toUnsignedString(value); + } + } + + @Override + public OceanBaseConstant castAsDouble() { + return this; + } + + @Override + public OceanBaseDataType getType() { + return OceanBaseDataType.INT; + } + + @Override + public boolean isSigned() { + return isSigned; + } + + private String getStringRepr() { + if (isSigned) { + return String.valueOf(value); + } else { + return Long.toUnsignedString(value); + } + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + if (rightVal.isInt()) { + long intVal = rightVal.getInt(); + if (isSigned && rightVal.isSigned()) { + return OceanBaseConstant.createBoolean(value < intVal); + } else { + return OceanBaseConstant.createBoolean(new BigInteger(getStringRepr()) + .compareTo(new BigInteger(((OceanBaseIntConstant) rightVal).getStringRepr())) < 0); + } + } else if (rightVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else if (rightVal.isString()) { + if (rightVal.asBooleanNotNull()) { + throw new IgnoreMeException(); + } + return isLessThan(rightVal.castAs(isSigned ? CastType.SIGNED : CastType.UNSIGNED)); + } else if (rightVal instanceof OceanBaseDoubleConstant){ + return OceanBaseConstant.createBoolean(value < rightVal.getDouble()); + } else { + throw new AssertionError(rightVal); + } + } + + } + + public static class OceanBaseNullConstant extends OceanBaseConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public boolean asBooleanNotNull() { + throw new UnsupportedOperationException(this.toString()); + } + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { + return OceanBaseConstant.createNullConstant(); + } + + @Override + public OceanBaseConstant castAs(CastType type) { + return this; + } + + @Override + public String castAsString() { + return "NULL"; + } + + @Override + public OceanBaseConstant castAsDouble() { + return this; + } + + @Override + public OceanBaseDataType getType() { + return null; + } + + @Override + protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { + return this; + } + + } + + public long getInt() { + throw new UnsupportedOperationException(); + } + + public double getDouble() { + throw new UnsupportedOperationException(); + } + + public boolean isSigned() { + return false; + } + + public String getString() { + throw new UnsupportedOperationException(); + } + + public boolean isString() { + return false; + } + + public static OceanBaseConstant createNullConstant() { + return new OceanBaseNullConstant(); + } + + public static OceanBaseConstant createIntConstant(long value) { + return new OceanBaseIntConstant(value, true); + } + + public static OceanBaseConstant createIntConstant(long value, boolean signed) { + return new OceanBaseIntConstant(value, signed); + } + + public static OceanBaseConstant createUnsignedIntConstant(long value) { + return new OceanBaseIntConstant(value, false); + } + + public static OceanBaseConstant createIntConstantNotAsBoolean(long value) { + return new OceanBaseIntConstant(value, String.valueOf(value)); + } + + public static OceanBaseConstant createDoubleConstant(double value) { + return new OceanBaseDoubleConstant(value); + } + + @Override + public OceanBaseConstant getExpectedValue() { + return this; + } + + public abstract boolean asBooleanNotNull(); + + public abstract String getTextRepresentation(); + + public static OceanBaseConstant createFalse() { + return OceanBaseConstant.createIntConstant(0); + } + + public static OceanBaseConstant createBoolean(boolean isTrue) { + return OceanBaseConstant.createIntConstant(isTrue ? 1 : 0); + } + + public static OceanBaseConstant createTrue() { + return OceanBaseConstant.createIntConstant(1); + } + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract OceanBaseConstant isEquals(OceanBaseConstant rightVal); + + public abstract OceanBaseConstant castAs(CastType type); + + public abstract String castAsString(); + + public abstract OceanBaseConstant castAsDouble(); + + public static OceanBaseConstant createStringConstant(String string) { + return new OceanBaseTextConstant(string); + } + + public abstract OceanBaseDataType getType(); + + protected abstract OceanBaseConstant isLessThan(OceanBaseConstant rightVal); + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseExists.java b/src/sqlancer/oceanbase/ast/OceanBaseExists.java new file mode 100644 index 000000000..18595d261 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseExists.java @@ -0,0 +1,30 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseExists implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final OceanBaseConstant expected; + + public OceanBaseExists(OceanBaseExpression expr, OceanBaseConstant expectedValue) { + this.expr = expr; + this.expected = expectedValue; + } + + public OceanBaseExists(OceanBaseExpression expr) { + this.expr = expr; + this.expected = expr.getExpectedValue(); + if (expected == null) { + throw new AssertionError(); + } + } + + public OceanBaseExpression getExpr() { + return expr; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return expected; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseExpression.java b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java new file mode 100644 index 000000000..195a05967 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java @@ -0,0 +1,9 @@ +package sqlancer.oceanbase.ast; + +public interface OceanBaseExpression { + + default OceanBaseConstant getExpectedValue() { + throw new AssertionError("PQS not supported for this operator"); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java new file mode 100644 index 000000000..e5c482a56 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java @@ -0,0 +1,58 @@ +package sqlancer.oceanbase.ast; + +import java.util.List; + +import sqlancer.IgnoreMeException; + +public class OceanBaseInOperation implements OceanBaseExpression { + + private final OceanBaseExpression expr; + private final List listElements; + private final boolean isTrue; + + public OceanBaseInOperation(OceanBaseExpression expr, List listElements, boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant leftVal = expr.getExpectedValue(); + if (leftVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } + boolean isNull = false; + for (OceanBaseExpression rightExpr : listElements) { + OceanBaseConstant rightVal = rightExpr.getExpectedValue(); + + OceanBaseConstant convertedRightVal = rightVal; + OceanBaseConstant isEquals = leftVal.isEquals(convertedRightVal); + if (isEquals.isNull()) { + isNull = true; + } else { + if (isEquals.getInt() == 1) { + return OceanBaseConstant.createBoolean(isTrue); + } + } + } + if (isNull) { + return OceanBaseConstant.createNullConstant(); + } else { + return OceanBaseConstant.createBoolean(!isTrue); + } + + } + + public boolean isTrue() { + return isTrue; + } +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseJoin.java b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java new file mode 100644 index 000000000..9e8271bbf --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java @@ -0,0 +1,10 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseJoin implements OceanBaseExpression { + + @Override + public OceanBaseConstant getExpectedValue() { + throw new UnsupportedOperationException(); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java b/src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java new file mode 100644 index 000000000..b8f2d029a --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseOrderByTerm.java @@ -0,0 +1,36 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.Randomly; + +public class OceanBaseOrderByTerm implements OceanBaseExpression { + + private final OceanBaseOrder order; + private final OceanBaseExpression expr; + + public enum OceanBaseOrder { + ASC, DESC; + + public static OceanBaseOrder getRandomOrder() { + return Randomly.fromOptions(OceanBaseOrder.values()); + } + } + + public OceanBaseOrderByTerm(OceanBaseExpression expr, OceanBaseOrder order) { + this.expr = expr; + this.order = order; + } + + public OceanBaseOrder getOrder() { + return order; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + @Override + public OceanBaseConstant getExpectedValue() { + throw new AssertionError(this); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java new file mode 100644 index 000000000..642553173 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java @@ -0,0 +1,58 @@ +package sqlancer.oceanbase.ast; + +import java.util.Collections; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.ast.SelectBase; + +public class OceanBaseSelect extends SelectBase implements OceanBaseExpression { + + private SelectType fromOptions = SelectType.ALL; + private List modifiers = Collections.emptyList(); + private List groupBys = new ArrayList<>(); + private OceanBaseStringExpression hint; + + public enum SelectType { + DISTINCT, ALL; + } + + public void setSelectType(SelectType fromOptions) { + this.setFromOptions(fromOptions); + } + + public SelectType getFromOptions() { + return fromOptions; + } + + public void setFromOptions(SelectType fromOptions) { + this.fromOptions = fromOptions; + } + public void setGroupByClause(List groupBys){ + this.groupBys = groupBys; + } + public List getGroupByClause(){ + return this.groupBys; + } + public void setModifiers(List modifiers) { + this.modifiers = modifiers; + } + + public List getModifiers() { + return modifiers; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return null; + } + + public void setHint(OceanBaseStringExpression hint) { + this.hint = hint; + } + + public OceanBaseStringExpression getHint() { + return hint; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java b/src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java new file mode 100644 index 000000000..3c1a55176 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseStringExpression.java @@ -0,0 +1,22 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseStringExpression implements OceanBaseExpression { + + private final String str; + private final OceanBaseConstant expectedValue; + + public OceanBaseStringExpression(String str, OceanBaseConstant expectedValue) { + this.str = str; + this.expectedValue = expectedValue; + } + + public String getStr() { + return str; + } + + @Override + public OceanBaseConstant getExpectedValue() { + return expectedValue; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseTableReference.java b/src/sqlancer/oceanbase/ast/OceanBaseTableReference.java new file mode 100644 index 000000000..8102ca1cc --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseTableReference.java @@ -0,0 +1,17 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseTableReference implements OceanBaseExpression { + + private final OceanBaseTable table; + + public OceanBaseTableReference(OceanBaseTable table) { + this.table = table; + } + + public OceanBaseTable getTable() { + return table; + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseText.java b/src/sqlancer/oceanbase/ast/OceanBaseText.java new file mode 100644 index 000000000..566effde4 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseText.java @@ -0,0 +1,27 @@ +package sqlancer.oceanbase.ast; + + +public class OceanBaseText implements OceanBaseExpression { + + private OceanBaseExpression expr; + private String text; + private boolean prefix; + + public OceanBaseText(OceanBaseExpression expr, String text, boolean prefix) { + this.expr = expr; + this.text = text; + this.prefix = prefix; + } + + public OceanBaseExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + public boolean isPrefix() { + return prefix; + } +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java new file mode 100644 index 000000000..a2c3b6a57 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPostfixOperation.java @@ -0,0 +1,58 @@ +package sqlancer.oceanbase.ast; + +public class OceanBaseUnaryPostfixOperation implements OceanBaseExpression { + + private final OceanBaseExpression expression; + private final UnaryPostfixOperator operator; + private boolean negate; + + public enum UnaryPostfixOperator { + IS_NULL, IS_TRUE, IS_FALSE; + } + + public OceanBaseUnaryPostfixOperation(OceanBaseExpression expr, UnaryPostfixOperator op, boolean negate) { + this.expression = expr; + this.operator = op; + this.setNegate(negate); + } + + public OceanBaseExpression getExpression() { + return expression; + } + + public UnaryPostfixOperator getOperator() { + return operator; + } + + public boolean isNegated() { + return negate; + } + + public void setNegate(boolean negate) { + this.negate = negate; + } + + @Override + public OceanBaseConstant getExpectedValue() { + boolean val; + OceanBaseConstant expectedValue = expression.getExpectedValue(); + switch (operator) { + case IS_NULL: + val = expectedValue.isNull(); + break; + case IS_FALSE: + val = !expectedValue.isNull() && !expectedValue.asBooleanNotNull(); + break; + case IS_TRUE: + val = !expectedValue.isNull() && expectedValue.asBooleanNotNull(); + break; + default: + throw new AssertionError(operator); + } + if (negate) { + val = !val; + } + return OceanBaseConstant.createIntConstant(val ? 1 : 0); + } + +} diff --git a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java new file mode 100644 index 000000000..c598e3bd0 --- /dev/null +++ b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java @@ -0,0 +1,91 @@ +package sqlancer.oceanbase.ast; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.UnaryOperatorNode; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; + +public class OceanBaseUnaryPrefixOperation extends UnaryOperatorNode + implements OceanBaseExpression { + + public enum OceanBaseUnaryPrefixOperator implements Operator { + NOT("!", "NOT") { + @Override + public OceanBaseConstant applyNotNull(OceanBaseConstant expr) { + return OceanBaseConstant.createIntConstant(expr.asBooleanNotNull() ? 0 : 1); + } + }, + PLUS("+") { + @Override + public OceanBaseConstant applyNotNull(OceanBaseConstant expr) { + return expr; + } + }, + MINUS("-") { + @Override + public OceanBaseConstant applyNotNull(OceanBaseConstant expr) { + if (expr.isString()) { + throw new IgnoreMeException(); + } else if (expr.isInt()) { + if (!expr.isSigned()) { + throw new IgnoreMeException(); + } + return OceanBaseConstant.createIntConstant(-expr.getInt()); + } else if (expr.isDouble()) { + return OceanBaseConstant.createDoubleConstant(-expr.getDouble()); + } else { + throw new AssertionError(expr); + } + } + }; + + private String[] textRepresentations; + + OceanBaseUnaryPrefixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public abstract OceanBaseConstant applyNotNull(OceanBaseConstant expr); + + public static OceanBaseUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + } + + public OceanBaseUnaryPrefixOperation(OceanBaseExpression expr, OceanBaseUnaryPrefixOperator op) { + super(expr, op); + } + + @Override + public OceanBaseConstant getExpectedValue() { + OceanBaseConstant subExprVal = expr.getExpectedValue(); + if (op == OceanBaseUnaryPrefixOperator.PLUS){ + if (subExprVal.isNull() && subExprVal.getType() == null) + return OceanBaseConstant.createNullConstant(); + else + return subExprVal; + } + if (subExprVal.isNull()) { + return OceanBaseConstant.createNullConstant(); + } else { + return op.applyNotNull(subExprVal); + } + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.PREFIX; + } + public OceanBaseExpression getExpr() { + return expr; + } + public OceanBaseUnaryPrefixOperator getOp() { + return op; + } +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java new file mode 100644 index 000000000..b3f7de7bb --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java @@ -0,0 +1,72 @@ +package sqlancer.oceanbase.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseAlterTable { + + private final OceanBaseSchema schema; + private final StringBuilder sb = new StringBuilder(); + boolean couldAffectSchema; + private List selectedActions; + + public OceanBaseAlterTable(OceanBaseSchema newSchema) { + this.schema = newSchema; + } + + public static SQLQueryAdapter create(OceanBaseGlobalState globalState) { + return new OceanBaseAlterTable(globalState.getSchema()).create(); + } + + private enum Action { + COMPRESSION; + private String[] potentialErrors; + + Action(String... couldCauseErrors) { + this.potentialErrors = couldCauseErrors.clone(); + } + + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = ExpectedErrors.from("does not support the create option", "doesn't have this option", + "is not supported for this operation", "Data truncation", "Specified key was too long"); + errors.add("Data truncated for functional index "); + sb.append("ALTER TABLE "); + OceanBaseTable table = schema.getRandomTable(); + sb.append(table.getName()); + sb.append(" "); + List list = new ArrayList<>(Arrays.asList(Action.values())); + selectedActions = Randomly.subset(list); + int i = 0; + for (Action a : selectedActions) { + if (i++ != 0) { + sb.append(", "); + } + switch (a) { + case COMPRESSION: + sb.append("COMPRESSION "); + sb.append("'"); + sb.append(Randomly.fromOptions("ZLIB_1.0", "LZ4_1.0", "NONE")); + sb.append("'"); + break; + } + } + for (Action a : selectedActions) { + for (String error : a.potentialErrors) { + errors.add(error); + } + } + return new SQLQueryAdapter(sb.toString(), errors, couldAffectSchema); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java new file mode 100644 index 000000000..de76917e8 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -0,0 +1,52 @@ +package sqlancer.oceanbase.gen; + +import java.util.Arrays; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseDeleteGenerator { + + private final StringBuilder sb = new StringBuilder(); + private final OceanBaseGlobalState globalState; + private final Randomly r; + public OceanBaseDeleteGenerator(OceanBaseGlobalState globalState) { + this.globalState = globalState; + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter delete(OceanBaseGlobalState globalState) { + return new OceanBaseDeleteGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + OceanBaseTable randomTable = globalState.getSchema().getRandomTable(); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(randomTable.getColumns()); + ExpectedErrors errors = new ExpectedErrors(); + sb.append("DELETE"); + if (Randomly.getBoolean()) { + sb.append(" /*+parallel("+r.getLong(0, 10)+") enable_parallel_dml*/ "); + } + sb.append(" FROM "); + sb.append(randomTable.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + OceanBaseErrors.addExpressionErrors(errors); + } + errors.addAll(Arrays.asList("doesn't have this option", + "Truncated incorrect DOUBLE value", + "Truncated incorrect INTEGER value", + "Truncated incorrect DECIMAL value", + "Data truncated for functional index", + "Incorrect value", "Out of range value for column", + "Data truncation: %s value is out of range in '%s'")); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java new file mode 100644 index 000000000..5e7c06fb2 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java @@ -0,0 +1,30 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public final class OceanBaseDropIndex { + + private OceanBaseDropIndex() { + } + + public static SQLQueryAdapter generate(OceanBaseGlobalState globalState) { + OceanBaseTable table = globalState.getSchema().getRandomTable(); + if (!table.hasIndexes()) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder(); + sb.append("DROP INDEX "); + sb.append(table.getRandomIndex().getIndexName()); + sb.append(" ON "); + sb.append(table.getName()); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("LOCK=NONE is not supported", "ALGORITHM=INPLACE is not supported", + "Data truncation", "Data truncated for functional index")); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java new file mode 100644 index 000000000..279223d73 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java @@ -0,0 +1,212 @@ +package sqlancer.oceanbase.gen; + +import java.util.ArrayList; +import java.util.List; + +import java.sql.Connection; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseRowValue; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation.BinaryComparisonOperator; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator; +import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction.OceanBaseFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseConstant.OceanBaseDoubleConstant; +import sqlancer.oceanbase.ast.OceanBaseExists; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; + +public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator { + + private OceanBaseGlobalState state; + private OceanBaseRowValue rowVal; + private Randomly r; + private List OceanBaseColumns = new ArrayList<>(); + + public OceanBaseExpressionGenerator(OceanBaseGlobalState state) { + this.state = state; + } + public OceanBaseExpressionGenerator setCon(Connection con) { + return this; + } + public OceanBaseExpressionGenerator setState(OceanBaseGlobalState state) { + this.state = state; + return this; + } + + public OceanBaseExpressionGenerator setOceanBaseColumns(List columns) { + this.OceanBaseColumns = columns; + return this; + } + public OceanBaseExpressionGenerator(Randomly r) { + this.r = r; + } + + public OceanBaseExpressionGenerator setRowVal(OceanBaseRowValue rowVal) { + this.rowVal = rowVal; + return this; + } + + private enum Actions { + COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, + BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, EXISTS; + } + + @Override + public OceanBaseExpression generateExpression(int depth) { + if (depth >= state.getOptions().getMaxExpressionDepth()) { + return generateLeafNode(); + } + switch (Randomly.fromOptions(Actions.values())) { + case COLUMN: + return generateColumn(); + case LITERAL: + return generateConstant(); + case UNARY_PREFIX_OPERATION: + OceanBaseExpression subExpr = generateExpression(depth + 1); + OceanBaseUnaryPrefixOperator random = OceanBaseUnaryPrefixOperator.getRandom(); + return new OceanBaseUnaryPrefixOperation(subExpr, random); + case UNARY_POSTFIX: + return new OceanBaseUnaryPostfixOperation(generateExpression(depth + 1), + Randomly.fromOptions(OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.values()), + Randomly.getBoolean()); + case COMPUTABLE_FUNCTION: + return getComputableFunction(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return new OceanBaseBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), + OceanBaseBinaryLogicalOperator.getRandom()); + case BINARY_COMPARISON_OPERATION: + return new OceanBaseBinaryComparisonOperation(generateExpression(depth + 1), generateExpression(depth + 1), + BinaryComparisonOperator.getRandom()); + case CAST: + return new OceanBaseCastOperation(generateExpression(depth + 1), OceanBaseCastOperation.CastType.getRandom()); + case IN_OPERATION: + OceanBaseExpression expr = generateExpression(depth + 1); + List rightList = new ArrayList<>(); + for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { + rightList.add(generateExpression(depth + 1)); + } + return new OceanBaseInOperation(expr, rightList, Randomly.getBoolean()); + case EXISTS: + return getExists(); + default: + throw new AssertionError(); + } + } + + private OceanBaseExpression getExists() { + if (Randomly.getBoolean()) { + return new OceanBaseExists(new OceanBaseStringExpression("SELECT 1", OceanBaseConstant.createTrue())); + } else { + return new OceanBaseExists(new OceanBaseStringExpression("SELECT 1 from dual wHERE FALSE", OceanBaseConstant.createFalse())); + } + } + + private OceanBaseExpression getComputableFunction(int depth) { + OceanBaseFunction func = OceanBaseFunction.getRandomFunction(); + int nrArgs = func.getNrArgs(); + if (func.isVariadic()) { + nrArgs += Randomly.smallNumber(); + } + OceanBaseExpression[] args = new OceanBaseExpression[nrArgs]; + for (int i = 0; i < args.length; i++) { + args[i] = generateExpression(depth + 1); + } + return new OceanBaseComputableFunction(func, args); + } + + private enum ConstantType { + INT, NULL, STRING, DOUBLE; + public static ConstantType[] valuesPQS() { + return new ConstantType[] { INT, NULL, STRING }; + } + } + + @Override + public OceanBaseExpression generateConstant() { + ConstantType[] values; + if (state.usesPQS()) { + values = ConstantType.valuesPQS(); + } else { + values = ConstantType.values(); + } + switch (Randomly.fromOptions(values)) { + case INT: + return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); + case NULL: + return OceanBaseConstant.createNullConstant(); + case STRING: + String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); + OceanBaseConstant createStringConstant = OceanBaseConstant.createStringConstant(string); + return createStringConstant; + case DOUBLE: + double val = state.getRandomly().getDouble(); + return new OceanBaseDoubleConstant(val); + default: + throw new AssertionError(); + } + } + + @Override + public OceanBaseExpression generateColumn() { + OceanBaseColumn c = Randomly.fromList(columns); + OceanBaseConstant val; + if (rowVal == null) { + val = OceanBaseConstant.createNullConstant(); + } else { + val = rowVal.getValues().get(c); + } + return OceanBaseColumnReference.create(c, val); + } + + public OceanBaseExpression generateConstant(OceanBaseColumn col) { + switch (col.getType().name()) { + case "INT": + return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); + case "NULL": + return OceanBaseConstant.createNullConstant(); + case "VARCHAR": + String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); + OceanBaseConstant createStringConstant = OceanBaseConstant.createStringConstant(string); + return createStringConstant; + case "DOUBLE": + double val = state.getRandomly().getDouble(); + return new OceanBaseDoubleConstant(val); + case "FLOAT": + val = state.getRandomly().getDouble(); + return new OceanBaseDoubleConstant(val); + case "DECIMAL": + val = state.getRandomly().getDouble(); + return new OceanBaseDoubleConstant(val); + default: + throw new AssertionError(); + } + } + + @Override + public OceanBaseExpression negatePredicate(OceanBaseExpression predicate) { + return new OceanBaseUnaryPrefixOperation(predicate, OceanBaseUnaryPrefixOperator.NOT); + } + + @Override + public OceanBaseExpression isNull(OceanBaseExpression expr) { + return new OceanBaseUnaryPostfixOperation(expr, OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + } +} + diff --git a/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java new file mode 100644 index 000000000..aca0511ce --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java @@ -0,0 +1,110 @@ +package sqlancer.oceanbase.gen; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseInsertGenerator { + + private final OceanBaseTable table; + private final StringBuilder sb = new StringBuilder(); + private final ExpectedErrors errors = new ExpectedErrors(); + private final OceanBaseGlobalState globalState; + private final Randomly r; + private int type = 0; + + public OceanBaseInsertGenerator(OceanBaseGlobalState globalState) { + this.globalState = globalState; + table = globalState.getSchema().getRandomTable(); + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter insertRow(OceanBaseGlobalState globalState) throws SQLException { + if (Randomly.getBoolean()) { + return new OceanBaseInsertGenerator(globalState).generateInsert(); + } else { + return new OceanBaseInsertGenerator(globalState).generateReplace(); + } + } + + private SQLQueryAdapter generateReplace() { + sb.append("REPLACE"); + type = 1; + return generateInto(); + + } + + private SQLQueryAdapter generateInsert() { + sb.append("INSERT"); + if (Randomly.getBoolean()) { + sb.append(" /*+parallel("+r.getLong(0, 10)+") enable_parallel_dml*/ "); + } + if (Randomly.getBoolean()) { + sb.append(" "); + } + return generateInto(); + } + + private SQLQueryAdapter generateInto() { + sb.append(" INTO "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(") "); + sb.append("VALUES"); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + int nrRows; + if (Randomly.getBoolean()) { + nrRows = 1; + } else { + nrRows = 1 + Randomly.smallNumber(); + } + for (int row = 0; row < nrRows; row++) { + if (row != 0) { + sb.append(", "); + } + sb.append("("); + for (int c = 0; c < columns.size(); c++) { + if (c != 0) { + sb.append(", "); + } + OceanBaseColumn col = columns.get(c); + sb.append(OceanBaseVisitor.asString(gen.generateConstant(col))); + + } + sb.append(")"); + } + if(Randomly.getBoolean() && type == 0){ + + List upcolumns = table.getRandomNonEmptyColumnSubset(); + if (upcolumns.size() > 0 ){ + sb.append(" ON DUPLICATE KEY UPDATE "); + + sb.append(upcolumns.get(0).getName()); + sb.append("="); + sb.append(gen.generateConstant(upcolumns.get(0))); + } + } + errors.add("doesn't have a default value"); + errors.add("Data truncation"); + errors.add("Incorrect integer value"); + errors.add("Duplicate entry"); + errors.add("Data truncated for functional index"); + errors.add("Data truncated for column"); + errors.add("cannot be null"); + errors.add("Incorrect decimal value"); + errors.add("Duplicated primary key"); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java new file mode 100644 index 000000000..bf4f679a9 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java @@ -0,0 +1,282 @@ +package sqlancer.oceanbase.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; + +public class OceanBaseTableGenerator { + + private final StringBuilder sb = new StringBuilder(); + private final boolean allowPrimaryKey; + private boolean setPrimaryKey; + private final String tableName; + private final Randomly r; + private int columnId; + private boolean tableHasNullableColumn; + private int keysSpecified; + private final List columns = new ArrayList<>(); + private final OceanBaseSchema schema; + private final OceanBaseGlobalState globalState; + + public OceanBaseTableGenerator(OceanBaseGlobalState globalState, String tableName) { + this.tableName = tableName; + this.r = globalState.getRandomly(); + this.schema = globalState.getSchema(); + allowPrimaryKey = Randomly.getBoolean(); + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(OceanBaseGlobalState globalState, String tableName) { + return new OceanBaseTableGenerator(globalState, tableName).create(); + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + + sb.append("CREATE"); + sb.append(" TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { + sb.append(" LIKE "); + sb.append(schema.getRandomTable().getName()); + return new SQLQueryAdapter(sb.toString(), true); + } else { + sb.append("("); + for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(); + } + sb.append(")"); + sb.append(" "); + appendTableOptions(); + appendPartitionOptions(); + addCommonErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + } + + private void addCommonErrors(ExpectedErrors list) { + list.add("doesn't have this option"); + list.add("must include all columns"); + list.add("not allowed type for this type of partitioning"); + list.add("doesn't support BLOB/TEXT columns"); + list.add("A BLOB field is not allowed in partition function"); + list.add("Too many keys specified; max 1 keys allowed"); + list.add("The total length of the partitioning fields is too large"); + } + + private enum PartitionOptions { + HASH, KEY + } + + private void appendPartitionOptions() { + sb.append(" PARTITION BY"); + switch (Randomly.fromOptions(PartitionOptions.values())) { + case HASH: + sb.append(" HASH("); + sb.append(Randomly.fromList(columns)); + sb.append(")"); + sb.append(" partitions "); + sb.append(r.getInteger(1, 20)); + break; + case KEY: + sb.append(" KEY"); + sb.append(" ("); + sb.append(Randomly.nonEmptySubset(columns).stream().collect(Collectors.joining(", "))); + sb.append(")"); + break; + default: + throw new AssertionError(); + } + } + + private enum TableOptions { + BS,BLOOM,AUTO_INCREMENT; + + public static List getRandomTableOptions() { + List options; + if (Randomly.getBooleanWithSmallProbability()) { + options = Randomly.subset(TableOptions.values()); + } else { + if (Randomly.getBoolean()) { + options = Collections.emptyList(); + } else { + options = Randomly.nonEmptySubset(Arrays.asList(TableOptions.values()), 0); + } + } + return options; + } + } + + private void appendTableOptions() { + List tableOptions = TableOptions.getRandomTableOptions(); + int i = 0; + for (TableOptions o : tableOptions) { + if (i++ != 0) { + sb.append(", "); + } + switch (o) { + case AUTO_INCREMENT: + sb.append("AUTO_INCREMENT = " + r.getPositiveInteger()); + break; + case BLOOM: + sb.append("USE_BLOOM_FILTER = "); + if(Randomly.getBoolean()) + sb.append(" FALSE "); + else + sb.append(" true "); + break; + case BS: + sb.append(" BLOCK_SIZE = "); + if(Randomly.getBoolean()) + sb.append(" 16384 "); + else + sb.append(" 32768 "); + break; + default: + throw new AssertionError(o); + } + } + } + + private void appendColumn() { + String columnName = DBMSCommon.createColumnName(columnId); + columns.add(columnName); + sb.append(columnName); + appendColumnDefinition(); + columnId++; + } + + private enum ColumnOptions { + NULL_OR_NOT_NULL, UNIQUE, COMMENT, PRIMARY_KEY + } + + private void appendColumnDefinition() { + sb.append(" "); + + OceanBaseDataType randomType = OceanBaseDataType.getRandom(globalState); + boolean isTextType = randomType == OceanBaseDataType.VARCHAR; + appendTypeString(randomType); + sb.append(" "); + boolean isNull = false; + boolean columnHasPrimaryKey = false; + + List columnOptions = Randomly.subset(ColumnOptions.values()); + if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { + tableHasNullableColumn = true; + } + if (isTextType) { + columnOptions.remove(ColumnOptions.PRIMARY_KEY); + columnOptions.remove(ColumnOptions.UNIQUE); + } + for (ColumnOptions o : columnOptions) { + sb.append(" "); + switch (o) { + case NULL_OR_NOT_NULL: + // PRIMARY KEYs cannot be NULL + if (!columnHasPrimaryKey) { + if (Randomly.getBoolean()) { + sb.append("NULL"); + } + tableHasNullableColumn = true; + isNull = true; + } else { + sb.append("NOT NULL"); + } + break; + case UNIQUE: + sb.append("UNIQUE"); + keysSpecified++; + if (Randomly.getBoolean()) { + sb.append(" KEY"); + } + break; + case COMMENT: + // TODO: generate randomly + sb.append(String.format("COMMENT '%s' ", "asdf")); + break; + case PRIMARY_KEY: + // PRIMARY KEYs cannot be NULL + if (allowPrimaryKey && !setPrimaryKey && !isNull) { + sb.append("PRIMARY KEY"); + setPrimaryKey = true; + columnHasPrimaryKey = true; + } + break; + default: + throw new AssertionError(); + } + } + + } + + private void appendTypeString(OceanBaseDataType randomType) { + switch (randomType) { + case DECIMAL: + sb.append("DECIMAL"); + optionallyAddPrecisionAndScale(sb); + break; + case INT: + sb.append(Randomly.fromOptions("TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT")); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(Randomly.getNotCachedInteger(0, 255)); + sb.append(")"); + } + break; + case VARCHAR: + sb.append(Randomly.fromOptions("VARCHAR(500)")); + break; + case FLOAT: + sb.append("FLOAT"); + optionallyAddPrecisionAndScale(sb); + break; + case DOUBLE: + sb.append(Randomly.fromOptions("DOUBLE", "FLOAT")); + optionallyAddPrecisionAndScale(sb); + break; + default: + throw new AssertionError(); + } + if (randomType.isNumeric()) { + if (Randomly.getBoolean() && randomType != OceanBaseDataType.INT) { + sb.append(" UNSIGNED"); + } + if (Randomly.getBoolean()) { + sb.append(" ZEROFILL"); + } + } + } + + public static void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean()) { + sb.append("("); + long m = Randomly.getNotCachedInteger(1, 53); + sb.append(m); + sb.append(", "); + long nCandidate = Randomly.getNotCachedInteger(1, 30); + long n = Math.min(nCandidate, m); + sb.append(n); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java new file mode 100644 index 000000000..04f5d3613 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseTruncateTableGenerator.java @@ -0,0 +1,18 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseGlobalState; + +public final class OceanBaseTruncateTableGenerator { + + private OceanBaseTruncateTableGenerator() { + } + + public static SQLQueryAdapter generate(OceanBaseGlobalState globalState) { + StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); + sb.append(globalState.getSchema().getRandomTable().getName()); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("doesn't have this option")); + } + +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java new file mode 100644 index 000000000..ff20328d5 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -0,0 +1,54 @@ +package sqlancer.oceanbase.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.Randomly; +import sqlancer.oceanbase.*; + + +import java.sql.SQLException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class OceanBaseUpdateGenerator { + private OceanBaseUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(OceanBaseGlobalState globalState) throws SQLException { + Randomly r = globalState.getRandomly(); + ExpectedErrors errors = new ExpectedErrors(); + OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); + StringBuilder sb = new StringBuilder("UPDATE "); + if (Randomly.getBoolean()) { + sb.append(" /*+parallel("+r.getInteger(0, 10)+") enable_parallel_dml*/ "); + } + sb.append(table.getName()); + sb.append(" SET "); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append("="); + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant(columns.get(i))); + } else { + sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + OceanBaseErrors.addExpressionErrors(errors); + } + } + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + OceanBaseErrors.addExpressionErrors(errors); + sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + errors.add("Data Too Long"); + } + errors.add("Duplicated primary key"); + OceanBaseErrors.addInsertErrors(errors); + + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java new file mode 100644 index 000000000..09daea34a --- /dev/null +++ b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java @@ -0,0 +1,150 @@ +package sqlancer.oceanbase.gen.datadef; + +import java.util.List; +import java.util.Set; +import java.util.HashSet; +import java.util.ArrayList; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; + +public class OceanBaseIndexGenerator { + + private final Randomly r; + private StringBuilder sb = new StringBuilder(); + private boolean columnIsPrimaryKey; + private boolean containsInPlace; + private OceanBaseSchema schema; + private List columns = new ArrayList<>(); + private final OceanBaseGlobalState globalState; + + public OceanBaseIndexGenerator(OceanBaseSchema schema, Randomly r, OceanBaseGlobalState globalState) { + this.schema = schema; + this.r = r; + this.globalState = globalState; + } + + public static SQLQueryAdapter create(OceanBaseGlobalState globalState) { + return new OceanBaseIndexGenerator(globalState.getSchema(), globalState.getRandomly(), globalState).create(); + } + + public SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + OceanBaseErrors.addExpressionErrors(errors); + sb.append("CREATE "); + sb.append("INDEX "); + sb.append(globalState.getSchema().getFreeIndexName()); + indexType(); + sb.append(" ON "); + OceanBaseTable table = schema.getRandomTable(); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(table.getName()); + sb.append("("); + List randomColumn = table.getRandomNonEmptyColumnSubset(); + int i = 0; + for (OceanBaseColumn c : randomColumn) { + if (i++ != 0) { + sb.append(", "); + } + if (c.isPrimaryKey()) { + columnIsPrimaryKey = true; + } + c.isPartioned = true; + columns.add(c); + sb.append(c.getName()); + if (Randomly.getBoolean() && c.getType() == OceanBaseDataType.VARCHAR) { + sb.append("("); + sb.append(r.getInteger(1, 5)); + sb.append(")"); + c.isPartioned = false; + } + } + sb.append(")"); + appendPartitionOptions(); + indexOption(); + String string = sb.toString(); + sb = new StringBuilder(); + errors.add("A primary key index cannot be invisible"); + errors.add("Functional index on a column is not supported. Consider using a regular index instead."); + errors.add("Incorrect usage of spatial/fulltext/hash index and explicit index order"); + errors.add("must include all columns"); + errors.add("cannot index the expression"); + errors.add("Data truncation: Truncated incorrect"); + errors.add("a disallowed function."); + errors.add("Data truncation"); + errors.add("Cannot create a functional index on an expression that returns a BLOB or TEXT."); + errors.add("used in key specification without a key length"); + errors.add("can't be used in key specification with the used table type"); + errors.add("Specified key was too long"); + errors.add("out of range"); + errors.add("Data truncated for functional index"); + errors.add("used in key specification without a key length"); + errors.add("Row size too large"); // seems to happen together with MIN_ROWS in the table declaration + return new SQLQueryAdapter(string, errors, true); + } + + private enum PartitionOptions { + HASH + } + private void appendPartitionOptions() { + if (Randomly.getBoolean()) { + return; + } + + OceanBaseColumn colIndex = Randomly.fromList(columns); + + if(colIndex.isPartioned == false){ + return; + } + + if(colIndex.getType() == OceanBaseDataType.VARCHAR){ + sb.append(" PARTITION BY"); + sb.append(" KEY"); + sb.append(" ("); + String name = colIndex.getName(); + sb.append(name); + sb.append(")"); + sb.append(" partitions "); + sb.append(r.getInteger(1, 20)); + } + else if(OceanBaseDataType.INT == colIndex.getType()){ + sb.append(" PARTITION BY"); + sb.append(" HASH("); + String name = colIndex.getName(); + sb.append(name); + sb.append(") "); + sb.append(" partitions "); + sb.append(r.getInteger(1, 20)); + } + else{ + return; + } + } + + private void indexOption() { + if (Randomly.getBoolean()) { + sb.append(" "); + } + } + + private void indexType() { + if (Randomly.getBoolean()) { + sb.append(" USING "); + sb.append(Randomly.fromOptions("BTREE", "HASH")); + } + } + + public void setNewSchema(OceanBaseSchema schema) { + this.schema = schema; + } +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java new file mode 100644 index 000000000..85748870e --- /dev/null +++ b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java @@ -0,0 +1,204 @@ +package sqlancer.oceanbase.oracle; + +import sqlancer.Randomly; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseVisitor; +import sqlancer.oceanbase.ast.*; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction; +import sqlancer.oceanbase.ast.OceanBaseComputableFunction.OceanBaseFunction; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import java.util.Random; + +public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { + + // SELECT COUNT(*) FROM t0 WHERE ; + // SELECT SUM(count) FROM (SELECT IS TRUE as count FROM t0); + // SELECT (SELECT COUNT(*) FROM t0 WHERE c0 IS NOT 0) = (SELECT COUNT(*) FROM + // (SELECT c0 is NOT 0 FROM t0)); + private OceanBaseSchema s; + private Randomly r; + private String firstQueryString; + private String secondQueryString; + private static final int NOT_FOUND = -1; + + public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + errors.add("is out of range"); + // regex + errors.add("unmatched parentheses"); + errors.add("nothing to repeat at offset"); + errors.add("missing )"); + errors.add("missing terminating ]"); + errors.add("range out of order in character class"); + errors.add("unrecognized character after "); + errors.add("Got error '(*VERB) not recognized or malformed"); + errors.add("must be followed by"); + errors.add("malformed number or name after"); + errors.add("digit expected after"); + } + + public void check() throws SQLException { + OceanBaseSchema.OceanBaseTable randomTable = s.getRandomTable(); + List columns = randomTable.getColumns(); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state).setColumns(columns); + OceanBaseExpression randomWhereCondition = gen.generateExpression(); + List groupBys = Collections.emptyList(); // getRandomExpressions(columns); + List tableList = (Arrays.asList(randomTable)).stream().map(t -> new OceanBaseTableReference(t)) + .collect(Collectors.toList()); + int firstCount = getFirstQueryCount(tableList, randomWhereCondition, groupBys); + int secondCount = getSecondQuery(tableList, randomWhereCondition, groupBys); + if (firstCount != secondCount && firstCount != NOT_FOUND && secondCount != NOT_FOUND) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, + firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + private int getSecondQuery(List tableList, OceanBaseExpression randomWhereCondition, + List groupBys) throws SQLException { + OceanBaseSelect select = new OceanBaseSelect(); + select.setGroupByClause(groupBys); + OceanBaseExpression expr = getTrueExpr(randomWhereCondition); + + OceanBaseText asText = new OceanBaseText(expr, " as count", false); + select.setFetchColumns(Arrays.asList(asText)); + select.setFromList(tableList); + select.setSelectType(OceanBaseSelect.SelectType.ALL); + int secondCount = 0; + + unoptimizedQueryString = "SELECT SUM(count) FROM (" + OceanBaseVisitor.asString(select) + ") as asdf"; + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + if (options.logEachSelect()) { + logger.writeCurrent(unoptimizedQueryString); + } + try { + rs = q.executeAndGet(state); + } catch (Exception e) { + throw new AssertionError(optimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(List tableList, OceanBaseExpression randomWhereCondition, + List groupBys) throws SQLException { + OceanBaseSelect select = new OceanBaseSelect(); + select.setGroupByClause(groupBys); + // SELECT COUNT(t1.c3) FROM t1 WHERE (- (t1.c2)); + // SELECT SUM(count) FROM (SELECT ((- (t1.c2)) IS TRUE) as count FROM t1);; + OceanBaseAggregate aggr = new OceanBaseAggregate( + new OceanBaseColumnName(new OceanBaseSchema.OceanBaseColumn("*", OceanBaseSchema.OceanBaseDataType.INT, false, 0, false)), + OceanBaseAggregate.OceanBaseAggregateFunction.COUNT); + select.setFetchColumns(Arrays.asList(aggr)); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + select.setSelectType(OceanBaseSelect.SelectType.ALL); + int firstCount = 0; + optimizedQueryString = OceanBaseVisitor.asString(select); + SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); + SQLancerResultSet rs; + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try { + rs = q.executeAndGet(state); + } catch (Exception e) { + throw new AssertionError(firstQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + firstCount += rs.getLong(1); + } + rs.close(); + return firstCount; + } + + private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition){ + int i = new Random().nextInt(8); + OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE,false); + + OceanBaseUnaryPostfixOperation isNotTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE,true); + + OceanBaseUnaryPostfixOperation isFalse= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE,false); + + OceanBaseUnaryPostfixOperation isNotFalse= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE,true); + + OceanBaseUnaryPostfixOperation isNULL= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL,false); + + OceanBaseUnaryPostfixOperation isNotNULL= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL,true); + + OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); + switch(i){ + case 0: + expr = isTrue; + break; + case 1: + expr = new OceanBaseUnaryPrefixOperation(new OceanBaseBinaryLogicalOperation(isFalse, isNULL, OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case 2: + expr = new OceanBaseUnaryPrefixOperation(new OceanBaseUnaryPrefixOperation(isTrue, OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case 3: + expr = new OceanBaseUnaryPrefixOperation(isNotTrue, OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case 4: + expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); + break; + case 5: + OceanBaseExpression[] args = new OceanBaseExpression[3]; + args[0] = OceanBaseConstant.createIntConstant(1); + args[1] = isTrue; + args[2] = OceanBaseConstant.createIntConstant(0); + expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); + break; + case 6: + OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; + ifArgs[0] = OceanBaseConstant.createNullConstant(); + ifArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); + break; + case 7: + OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; + coalesceArgs[0] = OceanBaseConstant.createNullConstant(); + coalesceArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); + break; + } + return expr; + } + + +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..0b9c04040 --- /dev/null +++ b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java @@ -0,0 +1,159 @@ +package sqlancer.oceanbase.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.ArrayList; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseRowValue; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTables; +import sqlancer.oceanbase.OceanBaseVisitor; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation.UnaryPostfixOperator; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; + +public class OceanBasePivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + private List columns; + + public OceanBasePivotedQuerySynthesisOracle(OceanBaseGlobalState globalState) throws SQLException { + super(globalState); + OceanBaseErrors.addExpressionErrors(errors); + errors.add("in 'order clause'"); + errors.add("value is out of range"); + } + + @Override + public Query getRectifiedQuery() throws SQLException { + OceanBaseTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); + List tables = randomFromTables.getTables(); + + OceanBaseSelect selectStatement = new OceanBaseSelect(); + selectStatement.setSelectType(Randomly.fromOptions(OceanBaseSelect.SelectType.values())); + columns = randomFromTables.getColumns(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + + selectStatement.setFromList(tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList())); + + fetchColumns = columns.stream().map(c -> new OceanBaseColumnReference(c, null)).map(d -> d.setRef(true)).collect(Collectors.toList()); + selectStatement.setFetchColumns(fetchColumns); + OceanBaseExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + OceanBaseExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + OceanBaseExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + List orderBy = generateOrderBy(columns); + selectStatement.setOrderByExpressions(orderBy); + + return new SQLQueryAdapter(OceanBaseVisitor.asString(selectStatement), errors); + } + + private List generateGroupByClause(List columns, OceanBaseRowValue rw) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> OceanBaseColumnReference.create(c, rw.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + public List generateOrderBy(List columns) { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new OceanBaseOrderByTerm(OceanBaseColumnReference.create(Randomly.fromList(columns), null), + OceanBaseOrder.getRandomOrder())); + } + return orderBys; + } + + private OceanBaseConstant generateLimit() { + if (Randomly.getBoolean()) { + return OceanBaseConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private OceanBaseExpression generateOffset() { + if (Randomly.getBoolean()) { + return OceanBaseConstant.createIntConstantNotAsBoolean(0); + } else { + return null; + } + } + + private OceanBaseExpression generateRectifiedExpression(List columns, OceanBaseRowValue rw) { + OceanBaseExpression expression = new OceanBaseExpressionGenerator(globalState).setRowVal(rw).setColumns(columns) + .generateExpression(); + OceanBaseConstant expectedValue = expression.getExpectedValue(); + OceanBaseExpression result; + if (expectedValue.isNull()) { + result = new OceanBaseUnaryPostfixOperation(expression, UnaryPostfixOperator.IS_NULL, false); + } else if (expectedValue.asBooleanNotNull()) { + result = expression; + } else { + result = new OceanBaseUnaryPrefixOperation(expression, OceanBaseUnaryPrefixOperator.NOT); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions + sb.append(query.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (OceanBaseColumn c : columns) { + if (i++ != 0) { + sb.append(" AND "); + } + if(pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) + sb.append("concat("); + sb.append("result." + c.getTable().getName() + c.getName()); + if(pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) + sb.append(",'')"); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); + } + } + + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, query.getExpectedErrors()); + } + + @Override + protected String getExpectedValues(OceanBaseExpression expr) { + return OceanBaseVisitor.asExpectedValues(expr); + } +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java new file mode 100644 index 000000000..3e05a305b --- /dev/null +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java @@ -0,0 +1,62 @@ +package sqlancer.oceanbase.oracle; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTables; +import sqlancer.oceanbase.ast.*; +import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; +import sqlancer.oceanbase.gen.OceanBaseHintGenerator; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; + +public abstract class OceanBaseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { + + OceanBaseSchema s; + OceanBaseTables targetTables; + OceanBaseExpressionGenerator gen; + OceanBaseSelect select; + + public OceanBaseTLPBase(OceanBaseGlobalState state) { + super(state); + OceanBaseErrors.addExpressionErrors(errors); + errors.add("value is out of range"); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new OceanBaseExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new OceanBaseSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + OceanBaseHintGenerator.generateHints(select, tables); + List tableList = tables.stream().map(t -> new OceanBaseTableReference(t)) + .collect(Collectors.toList()); + select.setFromList(tableList); + select.setWhereClause(null); + } + + List generateFetchColumns() { + return Arrays.asList(OceanBaseColumnReference.create(targetTables.getColumns().get(0), null)); + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java new file mode 100644 index 000000000..73a0c3a6f --- /dev/null +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java @@ -0,0 +1,44 @@ +package sqlancer.oceanbase.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseVisitor; + +public class OceanBaseTLPWhereOracle extends OceanBaseTLPBase { + + public OceanBaseTLPWhereOracle(OceanBaseGlobalState state) { + super(state); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = OceanBaseVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setOrderByExpressions(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = OceanBaseVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = OceanBaseVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = OceanBaseVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + +} diff --git a/test/sqlancer/dbms/TestOceanBaseNoREC.java b/test/sqlancer/dbms/TestOceanBaseNoREC.java new file mode 100644 index 000000000..3101c63b3 --- /dev/null +++ b/test/sqlancer/dbms/TestOceanBaseNoREC.java @@ -0,0 +1,29 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestOceanBaseNoREC { + + String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); + boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); + + @Test + public void testNoREC() { + assumeTrue(oceanBaseIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--database-prefix", + "pqsdb", "--num-queries", + TestConfig.NUM_QUERIES, + "--username","sqlancer@test", "--password", "sqlancer", + //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: + //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "oceanbase", "--oracle", "NoREC" })); + } + +} diff --git a/test/sqlancer/dbms/TestOceanBasePQS.java b/test/sqlancer/dbms/TestOceanBasePQS.java new file mode 100644 index 000000000..52dcb2eb5 --- /dev/null +++ b/test/sqlancer/dbms/TestOceanBasePQS.java @@ -0,0 +1,29 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestOceanBasePQS { + + String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); + boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); + + @Test + public void testPQS() { + assumeTrue(oceanBaseIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", + "pqsdb", "--num-queries", + TestConfig.NUM_QUERIES, + "--username","sqlancer@test", "--password", "sqlancer", + //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: + //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "oceanbase", "--oracle", "PQS" })); + } + +} diff --git a/test/sqlancer/dbms/TestOceanBaseTLP.java b/test/sqlancer/dbms/TestOceanBaseTLP.java new file mode 100644 index 000000000..e172c21fc --- /dev/null +++ b/test/sqlancer/dbms/TestOceanBaseTLP.java @@ -0,0 +1,29 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestOceanBaseTLP { + + String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); + boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); + + @Test + public void testTLP() { + assumeTrue(oceanBaseIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--database-prefix", + "pqsdb", "--num-queries", + TestConfig.NUM_QUERIES, + "--username","sqlancer@test", "--password", "sqlancer", + //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: + //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "oceanbase", "--oracle", "TLP_WHERE" })); + } + +} From 6b7c6f62caf10fab4a6195ef6367ea9ebe6f4f45 Mon Sep 17 00:00:00 2001 From: bashaojing Date: Wed, 15 Dec 2021 20:51:01 +0800 Subject: [PATCH 040/856] fix for hint,update default port,add readme for install oceanbase --- README.md | 1 + .../oceanbase/OceanBaseHintGenerator.java | 78 ++++++++++--------- src/sqlancer/oceanbase/OceanBaseOptions.java | 6 +- src/sqlancer/oceanbase/OceanBaseProvider.java | 6 ++ src/sqlancer/oceanbase/README.md | 21 +++++ test/sqlancer/dbms/TestOceanBaseNoREC.java | 2 +- test/sqlancer/dbms/TestOceanBasePQS.java | 2 +- test/sqlancer/dbms/TestOceanBaseTLP.java | 2 +- 8 files changed, 79 insertions(+), 39 deletions(-) create mode 100644 src/sqlancer/oceanbase/README.md diff --git a/README.md b/README.md index 9559ac048..7008c1d92 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,7 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | DuckDB | Working | Untyped, Generic | | | ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | | TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | +| OceanBase | Working | Untyped | | # Using SQLancer diff --git a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java index 83accab72..b3f9d5412 100644 --- a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java +++ b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java @@ -18,22 +18,24 @@ public class OceanBaseHintGenerator { private Randomly r = new Randomly(); enum IndexHint { + PDML, + NO_PRED_DEDUCE, MERGE_JOIN, - INL_JOIN, - INL_HASH_JOIN, - INL_MERGE_JOIN, - HASH_JOIN, - HASH_AGG, - STREAM_AGG, - USE_INDEX, - IGNORE_INDEX, - AGG_TO_COP, - USE_INDEX_MERGE, - NO_INDEX_MERGE, + HASH_JOIN, + NL_JOIN, + BNL_JOIN, + NO_MERGE_JOIN, + NO_HASH_JOIN, + NO_NL_JOIN, + NO_BNL_JOIN, + HASH_AGG, + NL_MATERIALIZATION, + LATE_MATERIALIZATION, + USE_INDEX, + TOPK, LEADING, - PredDeduce, - PDML, - USE_TOJA; + ORDERED, + NO_REWRITE; } public OceanBaseHintGenerator(OceanBaseSelect select, List tables) { @@ -52,50 +54,56 @@ private void generate() { case PDML: sb.append(" parallel(" + r.getInteger(0, 10) + "),enable_parallel_dml "); break; - case PredDeduce: - sb.append("no_pred_deduce"); + case NO_PRED_DEDUCE: + sb.append("NO_PRED_DEDUCE"); break; case MERGE_JOIN: tablesHint("USE_MERGE "); break; - case INL_JOIN: + case HASH_JOIN: + tablesHint("USE_HASH "); + break; + case NL_JOIN: tablesHint("USE_NL "); break; - case LEADING: - tablesHint(" LEADING "); + case BNL_JOIN: + tablesHint("USE_BNL "); break; - case INL_HASH_JOIN: - tablesHint("USE_HASH "); + case NO_MERGE_JOIN: + sb.append(" NO_USE_MERGE "); break; - case INL_MERGE_JOIN: - tablesHint("USE_BNL "); + case NO_HASH_JOIN: + sb.append(" NO_USE_HASH "); break; - case HASH_JOIN: - sb.append(" parallel(1) "); + case NO_NL_JOIN: + sb.append(" NO_USE_NL "); + break; + case NO_BNL_JOIN: + sb.append(" NO_USE_BNL "); break; case HASH_AGG: sb.append("USE_HASH_AGGREGATION "); break; - case STREAM_AGG: + case NL_MATERIALIZATION: sb.append("USE_NL_MATERIALIZATION "); break; + case LATE_MATERIALIZATION: + sb.append("USE_LATE_MATERIALIZATION "); + break; case USE_INDEX: indexesHint("INDEX_HINT "); break; - case IGNORE_INDEX: + case TOPK: sb.append("TOPK (50 50) "); break; - case AGG_TO_COP: - sb.append("USE_LATE_MATERIALIZATION "); + case LEADING: + tablesHint(" LEADING "); break; - case USE_INDEX_MERGE: + case ORDERED: sb.append("ORDERED "); break; - case NO_INDEX_MERGE: - tablesHint("NO_MERGE "); - break; - case USE_TOJA: - sb.append("no_rewrite " ); + case NO_REWRITE: + sb.append("NO_REWRITE " ); break; default: throw new AssertionError(); diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index fb529e375..bd4c20f46 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -19,7 +19,7 @@ + ", default host: " + OceanBaseOptions.DEFAULT_HOST) public class OceanBaseOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; - public static final int DEFAULT_PORT = 2883; + public static final int DEFAULT_PORT = 2881; @Parameter(names = "--oracle") public List oracles = Arrays.asList(OceanBaseOracleFactory.TLP_WHERE); @@ -51,6 +51,10 @@ public boolean requiresAllTablesToContainRows() { } } } + @Parameter(names = { "--query-timeout" }, description = "Query timeout") + public int queryTimeout = 1000000000; + @Parameter(names = { "--transaction-timeout" }, description = "Transaction timeout") + public int trxTimeout = 1000000000; @Override public List getTestOracleFactory() { diff --git a/src/sqlancer/oceanbase/OceanBaseProvider.java b/src/sqlancer/oceanbase/OceanBaseProvider.java index 5b57ee3c1..186109a70 100644 --- a/src/sqlancer/oceanbase/OceanBaseProvider.java +++ b/src/sqlancer/oceanbase/OceanBaseProvider.java @@ -140,6 +140,12 @@ public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exc String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", host, port); Connection con = DriverManager.getConnection(url, username, password); + try(Statement s = con.createStatement()){ + s.execute("set ob_query_timeout=" + globalState.getDbmsSpecificOptions().queryTimeout); + } + try(Statement s = con.createStatement()){ + s.execute("set ob_trx_timeout=" + globalState.getDbmsSpecificOptions().trxTimeout); + } try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); diff --git a/src/sqlancer/oceanbase/README.md b/src/sqlancer/oceanbase/README.md new file mode 100644 index 000000000..17940c678 --- /dev/null +++ b/src/sqlancer/oceanbase/README.md @@ -0,0 +1,21 @@ +## Install Oceanbase +There are some methods to install OceanBase. +A method to install a local single-node OceanBase cluster: +```shell +sudo yum install -y yum-utils +sudo yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo +sudo yum install -y ob-deploy +source /etc/profile.d/obd.sh + +git clone https://github.com/oceanbase/obdeploy.git +cd obdeploy +sudo obd cluster deploy c1 -c ./example/mini-local-example.yaml -A +sudo obd cluster start c1 +``` +> **NOTE:** We do not recommend that you use sys tenant to test. So please deploy clutser with optition -A, will create the test tenant during the bootstrap by using all available resources of the cluster. +> **NOTE:** Then you can create user. + +```shell +mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" +``` +Other methods, see [OceanBase Deployer](https://github.com/oceanbase/obdeploy/blob/master/README.md). diff --git a/test/sqlancer/dbms/TestOceanBaseNoREC.java b/test/sqlancer/dbms/TestOceanBaseNoREC.java index 3101c63b3..ddf6bcda0 100644 --- a/test/sqlancer/dbms/TestOceanBaseNoREC.java +++ b/test/sqlancer/dbms/TestOceanBaseNoREC.java @@ -22,7 +22,7 @@ public void testNoREC() { TestConfig.NUM_QUERIES, "--username","sqlancer@test", "--password", "sqlancer", //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "NoREC" })); } diff --git a/test/sqlancer/dbms/TestOceanBasePQS.java b/test/sqlancer/dbms/TestOceanBasePQS.java index 52dcb2eb5..4123e76c0 100644 --- a/test/sqlancer/dbms/TestOceanBasePQS.java +++ b/test/sqlancer/dbms/TestOceanBasePQS.java @@ -22,7 +22,7 @@ public void testPQS() { TestConfig.NUM_QUERIES, "--username","sqlancer@test", "--password", "sqlancer", //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "PQS" })); } diff --git a/test/sqlancer/dbms/TestOceanBaseTLP.java b/test/sqlancer/dbms/TestOceanBaseTLP.java index e172c21fc..6eef3fc91 100644 --- a/test/sqlancer/dbms/TestOceanBaseTLP.java +++ b/test/sqlancer/dbms/TestOceanBaseTLP.java @@ -22,7 +22,7 @@ public void testTLP() { TestConfig.NUM_QUERIES, "--username","sqlancer@test", "--password", "sqlancer", //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2883 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "TLP_WHERE" })); } From 9b02f1324f07c6b8a82c73f93f7940d65208e78b Mon Sep 17 00:00:00 2001 From: bashaojing Date: Thu, 16 Dec 2021 19:49:32 +0800 Subject: [PATCH 041/856] fix for format check and some comments --- .travis.yml | 6 - src/sqlancer/Main.java | 2 +- src/sqlancer/oceanbase/OceanBaseErrors.java | 6 +- .../OceanBaseExpectedValueVisitor.java | 8 +- .../oceanbase/OceanBaseHintGenerator.java | 141 --------------- src/sqlancer/oceanbase/OceanBaseOptions.java | 3 +- src/sqlancer/oceanbase/OceanBaseProvider.java | 30 ++-- src/sqlancer/oceanbase/OceanBaseSchema.java | 61 +++---- .../oceanbase/OceanBaseToStringVisitor.java | 42 ++--- .../OceanBaseUserCheckException.java | 8 + src/sqlancer/oceanbase/OceanBaseVisitor.java | 8 +- src/sqlancer/oceanbase/README.md | 18 +- .../oceanbase/ast/OceanBaseAggregate.java | 5 +- .../OceanBaseBinaryComparisonOperation.java | 5 +- .../ast/OceanBaseBinaryLogicalOperation.java | 5 +- .../oceanbase/ast/OceanBaseColumnName.java | 2 +- .../ast/OceanBaseColumnReference.java | 15 +- .../ast/OceanBaseComputableFunction.java | 95 +++++----- .../oceanbase/ast/OceanBaseConstant.java | 90 +++++----- .../oceanbase/ast/OceanBaseInOperation.java | 2 - .../oceanbase/ast/OceanBaseSelect.java | 9 +- src/sqlancer/oceanbase/ast/OceanBaseText.java | 7 +- .../ast/OceanBaseUnaryPrefixOperation.java | 9 +- .../oceanbase/gen/OceanBaseAlterTable.java | 4 +- .../gen/OceanBaseDeleteGenerator.java | 15 +- .../oceanbase/gen/OceanBaseDropIndex.java | 6 +- .../gen/OceanBaseExpressionGenerator.java | 54 +++--- .../oceanbase/gen/OceanBaseHintGenerator.java | 125 +++++++++++++ .../gen/OceanBaseInsertGenerator.java | 13 +- .../gen/OceanBaseTableGenerator.java | 57 +++--- .../gen/OceanBaseUpdateGenerator.java | 35 ++-- .../gen/datadef/OceanBaseIndexGenerator.java | 28 +-- .../oracle/OceanBaseNoRECOracle.java | 170 ++++++++++-------- .../OceanBasePivotedQuerySynthesisOracle.java | 24 +-- .../oceanbase/oracle/OceanBaseTLPBase.java | 20 +-- test/sqlancer/dbms/TestOceanBaseNoREC.java | 12 +- test/sqlancer/dbms/TestOceanBasePQS.java | 13 +- test/sqlancer/dbms/TestOceanBaseTLP.java | 12 +- 38 files changed, 590 insertions(+), 575 deletions(-) delete mode 100644 src/sqlancer/oceanbase/OceanBaseHintGenerator.java create mode 100644 src/sqlancer/oceanbase/OceanBaseUserCheckException.java create mode 100644 src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java diff --git a/.travis.yml b/.travis.yml index 6e5594948..318ea6fa7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -144,9 +144,3 @@ matrix: - sleep 5 script: - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test - - name: OceanBase - jdk : openjdk8 - script: - - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBaseTLP test - - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBasePQS test - - OCEANBASE_AVAILABLE=true mvn -Dtest=TestOceanBaseNoREC test diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 1ab9e29d9..a3ff789e4 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -34,10 +34,10 @@ import sqlancer.mariadb.MariaDBProvider; import sqlancer.mongodb.MongoDBProvider; import sqlancer.mysql.MySQLProvider; +import sqlancer.oceanbase.OceanBaseProvider; import sqlancer.postgres.PostgresProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; -import sqlancer.oceanbase.OceanBaseProvider; public final class Main { diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index 392bd7c5e..893490c34 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -16,6 +16,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Invalid numeric"); errors.add("Data truncated for argument"); } + public static void addInsertErrors(ExpectedErrors errors) { errors.add("Duplicate entry"); errors.add("cannot be null"); @@ -39,10 +40,7 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Truncated incorrect DOUBLE value"); errors.add("Data truncated for argument"); errors.add("Invalid numeric"); + errors.add("Miss column"); - - if (true) { - errors.add("Miss column"); - } } } diff --git a/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java index 2c03436d1..f9e57bd24 100644 --- a/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseExpectedValueVisitor.java @@ -1,9 +1,11 @@ package sqlancer.oceanbase; import sqlancer.IgnoreMeException; +import sqlancer.oceanbase.ast.OceanBaseAggregate; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseColumnReference; import sqlancer.oceanbase.ast.OceanBaseComputableFunction; import sqlancer.oceanbase.ast.OceanBaseConstant; @@ -14,10 +16,8 @@ import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseStringExpression; import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; -import sqlancer.oceanbase.ast.OceanBaseAggregate; -import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; public class OceanBaseExpectedValueVisitor implements OceanBaseVisitor { @@ -147,7 +147,7 @@ public void visit(OceanBaseText func) { } @Override - public void visit(OceanBaseUnaryPrefixOperation op){ + public void visit(OceanBaseUnaryPrefixOperation op) { print(op); visit(op.getExpr()); } diff --git a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java b/src/sqlancer/oceanbase/OceanBaseHintGenerator.java deleted file mode 100644 index b3f9d5412..000000000 --- a/src/sqlancer/oceanbase/OceanBaseHintGenerator.java +++ /dev/null @@ -1,141 +0,0 @@ -package sqlancer.oceanbase.gen; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.ast.OceanBaseConstant; -import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.ast.OceanBaseStringExpression; - -import java.util.List; -import java.util.stream.Collectors; - -public class OceanBaseHintGenerator { - private OceanBaseSelect select; - private List tables; - private final StringBuilder sb = new StringBuilder(); - private Randomly r = new Randomly(); - - enum IndexHint { - PDML, - NO_PRED_DEDUCE, - MERGE_JOIN, - HASH_JOIN, - NL_JOIN, - BNL_JOIN, - NO_MERGE_JOIN, - NO_HASH_JOIN, - NO_NL_JOIN, - NO_BNL_JOIN, - HASH_AGG, - NL_MATERIALIZATION, - LATE_MATERIALIZATION, - USE_INDEX, - TOPK, - LEADING, - ORDERED, - NO_REWRITE; - } - - public OceanBaseHintGenerator(OceanBaseSelect select, List tables) { - this.select = select; - this.tables = tables; - } - - public static void generateHints(OceanBaseSelect select, List tables) { - new OceanBaseHintGenerator(select, tables).generate(); - - } - - private void generate() { - OceanBaseTable table = Randomly.fromList(tables); - switch (Randomly.fromOptions(IndexHint.values())) { - case PDML: - sb.append(" parallel(" + r.getInteger(0, 10) + "),enable_parallel_dml "); - break; - case NO_PRED_DEDUCE: - sb.append("NO_PRED_DEDUCE"); - break; - case MERGE_JOIN: - tablesHint("USE_MERGE "); - break; - case HASH_JOIN: - tablesHint("USE_HASH "); - break; - case NL_JOIN: - tablesHint("USE_NL "); - break; - case BNL_JOIN: - tablesHint("USE_BNL "); - break; - case NO_MERGE_JOIN: - sb.append(" NO_USE_MERGE "); - break; - case NO_HASH_JOIN: - sb.append(" NO_USE_HASH "); - break; - case NO_NL_JOIN: - sb.append(" NO_USE_NL "); - break; - case NO_BNL_JOIN: - sb.append(" NO_USE_BNL "); - break; - case HASH_AGG: - sb.append("USE_HASH_AGGREGATION "); - break; - case NL_MATERIALIZATION: - sb.append("USE_NL_MATERIALIZATION "); - break; - case LATE_MATERIALIZATION: - sb.append("USE_LATE_MATERIALIZATION "); - break; - case USE_INDEX: - indexesHint("INDEX_HINT "); - break; - case TOPK: - sb.append("TOPK (50 50) "); - break; - case LEADING: - tablesHint(" LEADING "); - break; - case ORDERED: - sb.append("ORDERED "); - break; - case NO_REWRITE: - sb.append("NO_REWRITE " ); - break; - default: - throw new AssertionError(); - } - - select.setHint(new OceanBaseStringExpression(sb.toString(),new OceanBaseConstant.OceanBaseTextConstant(sb.toString()))); - } - - private void indexesHint(String string) { - sb.append(string); - sb.append("("); - OceanBaseTable table = Randomly.fromList(tables); - List allIndexes = table.getIndexes(); - if (allIndexes.isEmpty()) { - throw new IgnoreMeException(); - } - List indexSubset = Randomly.nonEmptySubset(allIndexes); - sb.append(table.getName()); - sb.append(", "); - sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); - sb.append(")"); - } - - private void tablesHint(String string) { - sb.append(string); - sb.append("("); - appendTables(); - sb.append(")"); - } - - private void appendTables() { - List tableSubset = Randomly.nonEmptySubset(tables); - sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); - } -} diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index bd4c20f46..eb7200437 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -11,9 +11,9 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; +import sqlancer.oceanbase.oracle.OceanBaseNoRECOracle; import sqlancer.oceanbase.oracle.OceanBasePivotedQuerySynthesisOracle; import sqlancer.oceanbase.oracle.OceanBaseTLPWhereOracle; -import sqlancer.oceanbase.oracle.OceanBaseNoRECOracle; @Parameters(separators = "=", commandDescription = "OceanBase (default port: " + OceanBaseOptions.DEFAULT_PORT + ", default host: " + OceanBaseOptions.DEFAULT_HOST) @@ -51,6 +51,7 @@ public boolean requiresAllTablesToContainRows() { } } } + @Parameter(names = { "--query-timeout" }, description = "Query timeout") public int queryTimeout = 1000000000; @Parameter(names = { "--transaction-timeout" }, description = "Transaction timeout") diff --git a/src/sqlancer/oceanbase/OceanBaseProvider.java b/src/sqlancer/oceanbase/OceanBaseProvider.java index 186109a70..d7308ab63 100644 --- a/src/sqlancer/oceanbase/OceanBaseProvider.java +++ b/src/sqlancer/oceanbase/OceanBaseProvider.java @@ -31,21 +31,17 @@ public OceanBaseProvider() { } enum Action implements AbstractAction { - SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), - INSERT(OceanBaseInsertGenerator::insertRow), - CREATE_INDEX(OceanBaseIndexGenerator::create), - ALTER_TABLE(OceanBaseAlterTable::create), - TRUNCATE_TABLE(OceanBaseTruncateTableGenerator::generate), + SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), INSERT(OceanBaseInsertGenerator::insertRow), + CREATE_INDEX(OceanBaseIndexGenerator::create), ALTER_TABLE(OceanBaseAlterTable::create), + TRUNCATE_TABLE(OceanBaseTruncateTableGenerator::generate), SELECT_INFO((g) -> new SQLQueryAdapter( "select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + g.getDatabaseName() - + "'")), + + "'")), CREATE_TABLE((g) -> { String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); - + return OceanBaseTableGenerator.generate(g, tableName); - }), - DELETE(OceanBaseDeleteGenerator::delete), - UPDATE(OceanBaseUpdateGenerator::getQuery), + }), DELETE(OceanBaseDeleteGenerator::delete), UPDATE(OceanBaseUpdateGenerator::update), DROP_INDEX(OceanBaseDropIndex::generate); private final SQLQueryProvider sqlQueryProvider; @@ -118,7 +114,7 @@ public void generateDatabase(OceanBaseGlobalState globalState) throws Exception } @Override - public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exception,SQLException { + public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exception, SQLException { String username = globalState.getOptions().getUserName(); String password = globalState.getOptions().getPassword(); String host = globalState.getOptions().getHost(); @@ -129,9 +125,9 @@ public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exc if (port == MainOptions.NO_SET_PORT) { port = OceanBaseOptions.DEFAULT_PORT; } - if(username.endsWith("sys")||username.equals("root")) - { - throw new Exception("please don't use sys tenant to test! Firstly create tenant then test"); + if (username.endsWith("sys") || username.equals("root")) { + throw new OceanBaseUserCheckException( + "please don't use sys tenant to test! Firstly create tenant then test"); } String databaseName = globalState.getDatabaseName(); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); @@ -140,13 +136,13 @@ public SQLConnection createDatabase(OceanBaseGlobalState globalState) throws Exc String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", host, port); Connection con = DriverManager.getConnection(url, username, password); - try(Statement s = con.createStatement()){ + + try (Statement s = con.createStatement()) { s.execute("set ob_query_timeout=" + globalState.getDbmsSpecificOptions().queryTimeout); } - try(Statement s = con.createStatement()){ + try (Statement s = con.createStatement()) { s.execute("set ob_trx_timeout=" + globalState.getDbmsSpecificOptions().trxTimeout); } - try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); } diff --git a/src/sqlancer/oceanbase/OceanBaseSchema.java b/src/sqlancer/oceanbase/OceanBaseSchema.java index a16358110..7de2457ee 100644 --- a/src/sqlancer/oceanbase/OceanBaseSchema.java +++ b/src/sqlancer/oceanbase/OceanBaseSchema.java @@ -8,11 +8,10 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.stream.Stream; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.IgnoreMeException; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; @@ -65,11 +64,12 @@ public enum CollateSequence { public static CollateSequence random() { return Randomly.fromOptions(values()); - + } } - public OceanBaseColumn(String name, OceanBaseDataType columnType, boolean isPrimaryKey, int precision, boolean isZeroFill) { + public OceanBaseColumn(String name, OceanBaseDataType columnType, boolean isPrimaryKey, int precision, + boolean isZeroFill) { super(name, null, columnType); this.isPrimaryKey = isPrimaryKey; this.precision = precision; @@ -84,7 +84,7 @@ public int getPrecision() { public boolean isPrimaryKey() { return isPrimaryKey; } - + public boolean isZeroFill() { return isZeroFill; } @@ -98,10 +98,14 @@ public OceanBaseTables(List tables) { } public OceanBaseRowValue getRandomRowValue(SQLConnection con) throws SQLException { - String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", columnNamesAsString( - c -> c.getType()==OceanBaseDataType.FLOAT || c.isZeroFill() ? ("concat(" + c.getTable().getName() + "." + c.getName() + ",'')" + " AS " + c.getTable().getName() + c.getName()) : (c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName())), + String randomRow = String.format("SELECT %s FROM %s ORDER BY RAND() LIMIT 1", + columnNamesAsString(c -> c.getType() == OceanBaseDataType.FLOAT || c.isZeroFill() + ? "concat(" + c.getTable().getName() + "." + c.getName() + ",'')" + " AS " + + c.getTable().getName() + c.getName() + : c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + + c.getName()), tableNamesAsString()); - //cast float and zerofill as varchar + // cast float and zerofill as varchar Map values = new HashMap<>(); try (Statement s = con.createStatement()) { ResultSet randomRowValues = s.executeQuery(randomRow); @@ -114,20 +118,20 @@ public OceanBaseRowValue getRandomRowValue(SQLConnection con) throws SQLExceptio int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); assert columnIndex == i + 1; OceanBaseConstant constant; - if (randomRowValues.getString(columnIndex) == null) { - if(column.isZeroFill()) - constant= OceanBaseConstant.createStringConstant("null"); - else - constant = OceanBaseConstant.createNullConstant(); - + if (randomRowValues.getString(columnIndex) == null) { + if (column.isZeroFill()) { + constant = OceanBaseConstant.createStringConstant("null"); + } else { + constant = OceanBaseConstant.createNullConstant(); + } } else { switch (column.getType()) { case INT: - //cast zerofill as varchar - if (column.isZeroFill()){ + // cast zerofill as varchar + if (column.isZeroFill()) { value = randomRowValues.getString(columnIndex); constant = OceanBaseConstant.createStringConstant((String) value); - }else{ + } else { value = randomRowValues.getLong(columnIndex); constant = OceanBaseConstant.createIntConstant((long) value); } @@ -175,7 +179,8 @@ private static OceanBaseDataType getColumnType(String typeString) { } } - public static class OceanBaseRowValue extends AbstractRowValue { + public static class OceanBaseRowValue + extends AbstractRowValue { OceanBaseRowValue(OceanBaseTables tables, Map values) { super(tables, values); @@ -183,7 +188,8 @@ public static class OceanBaseRowValue extends AbstractRowValue { + public static class OceanBaseTable + extends AbstractRelationalTable { public OceanBaseTable(String tableName, List columns, List indexes) { super(tableName, columns, indexes, false); @@ -204,13 +210,6 @@ private OceanBaseIndex(String indexName) { public static OceanBaseIndex create(String indexName) { return new OceanBaseIndex(indexName); } - - @Override - public String getIndexName() { - return super.getIndexName(); - - } - } public static OceanBaseSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { @@ -219,8 +218,8 @@ public static OceanBaseSchema fromConnection(SQLConnection con, String databaseN try { List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery( - "select TABLE_NAME from information_schema.TABLES where table_schema = '" + try (ResultSet rs = s + .executeQuery("select TABLE_NAME from information_schema.TABLES where table_schema = '" + databaseName + "';")) { while (rs.next()) { String tableName = rs.getString("TABLE_NAME"); @@ -251,8 +250,9 @@ private static List getIndexes(SQLConnection con, String tableNa databaseName, tableName))) { while (rs.next()) { String indexName = rs.getString("INDEX_NAME"); - if(!indexName.equals("PRIMARY")) + if (!indexName.equals("PRIMARY")) { indexes.add(OceanBaseIndex.create(indexName)); + } } } } @@ -272,7 +272,8 @@ private static List getTableColumns(SQLConnection con, String t boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); boolean isZeroFill = rs.getString("COLUMN_TYPE").contains("zerofill"); - OceanBaseColumn c = new OceanBaseColumn(columnName, getColumnType(dataType), isPrimaryKey, precision, isZeroFill); + OceanBaseColumn c = new OceanBaseColumn(columnName, getColumnType(dataType), isPrimaryKey, + precision, isZeroFill); columns.add(c); } } diff --git a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java index 053731852..56fef98f1 100644 --- a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java @@ -5,9 +5,12 @@ import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; +import sqlancer.oceanbase.ast.OceanBaseAggregate; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseColumnReference; import sqlancer.oceanbase.ast.OceanBaseComputableFunction; import sqlancer.oceanbase.ast.OceanBaseConstant; @@ -19,17 +22,14 @@ import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseStringExpression; import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; -import sqlancer.oceanbase.ast.OceanBaseAggregate; -import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; public class OceanBaseToStringVisitor extends ToStringVisitor implements OceanBaseVisitor { int ref; - private Randomly r = new Randomly(); + private final Randomly r = new Randomly(); @Override public void visitSpecific(OceanBaseExpression expr) { @@ -41,7 +41,7 @@ public void visit(OceanBaseSelect s) { sb.append("SELECT "); if (s.getHint() != null) { sb.append("/*+ "); - visit(s.getHint(),0); + visit(s.getHint(), 0); sb.append(" */ "); } @@ -133,12 +133,14 @@ public String get() { @Override public void visit(OceanBaseColumnReference column) { - if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) - {sb.append("concat(");} + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) { + sb.append("concat("); + } sb.append(column.getColumn().getFullQualifiedName()); - if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) - {sb.append(",'')");} - if (column.getRef()){ + if (column.getColumn().getType() == OceanBaseDataType.FLOAT || column.getColumn().isZeroFill()) { + sb.append(",'')"); + } + if (column.getRef()) { sb.append(" AS "); sb.append(column.getColumn().getTable().getName()); sb.append(column.getColumn().getName()); @@ -254,21 +256,19 @@ public void visit(OceanBaseExists op) { @Override public void visit(OceanBaseStringExpression op) { - if(op.getStr().contains("SELECT")){ - sb.append(op.getStr()); - } - else{ + if (op.getStr().contains("SELECT")) { + sb.append(op.getStr()); + } else { String str = op.getStr(); - if(str.length() > 0){ + if (str.length() > 0) { sb.append(r.getInteger(0, 100000)); - } - else{ + } else { sb.append(r.getInteger(0, 1000000)); } } } - public void visit(OceanBaseStringExpression op,int type) { + public void visit(OceanBaseStringExpression op, int type) { sb.append(op.getStr()); } @@ -283,11 +283,13 @@ public void visit(OceanBaseAggregate aggr) { sb.append("("); visit(aggr.getExpr()); sb.append(")"); - } + } + @Override public void visit(OceanBaseColumnName c) { sb.append(c.getColumn().getName()); } + @Override public void visit(OceanBaseText func) { visit(func.getExpr()); diff --git a/src/sqlancer/oceanbase/OceanBaseUserCheckException.java b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java new file mode 100644 index 000000000..3aedc6f48 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java @@ -0,0 +1,8 @@ +package sqlancer.oceanbase; + +public class OceanBaseUserCheckException extends RuntimeException { + + public OceanBaseUserCheckException(String s) { + super(s); + } +} diff --git a/src/sqlancer/oceanbase/OceanBaseVisitor.java b/src/sqlancer/oceanbase/OceanBaseVisitor.java index 5babfd7df..91f3549f7 100644 --- a/src/sqlancer/oceanbase/OceanBaseVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseVisitor.java @@ -1,8 +1,10 @@ package sqlancer.oceanbase; +import sqlancer.oceanbase.ast.OceanBaseAggregate; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; import sqlancer.oceanbase.ast.OceanBaseCastOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseColumnReference; import sqlancer.oceanbase.ast.OceanBaseComputableFunction; import sqlancer.oceanbase.ast.OceanBaseConstant; @@ -13,10 +15,8 @@ import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseStringExpression; import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; -import sqlancer.oceanbase.ast.OceanBaseAggregate; -import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; public interface OceanBaseVisitor { @@ -52,7 +52,7 @@ public interface OceanBaseVisitor { void visit(OceanBaseColumnName c); void visit(OceanBaseText fun); - + void visit(OceanBaseUnaryPrefixOperation op); default void visit(OceanBaseExpression expr) { diff --git a/src/sqlancer/oceanbase/README.md b/src/sqlancer/oceanbase/README.md index 17940c678..ddac91aed 100644 --- a/src/sqlancer/oceanbase/README.md +++ b/src/sqlancer/oceanbase/README.md @@ -1,19 +1,33 @@ ## Install Oceanbase There are some methods to install OceanBase. A method to install a local single-node OceanBase cluster: +### Install OBD by using RPM packages (only for CentOS 7 or later) ```shell sudo yum install -y yum-utils sudo yum-config-manager --add-repo https://mirrors.aliyun.com/oceanbase/OceanBase.repo sudo yum install -y ob-deploy source /etc/profile.d/obd.sh - +``` +### Start an OceanBase cluster +```shell git clone https://github.com/oceanbase/obdeploy.git cd obdeploy sudo obd cluster deploy c1 -c ./example/mini-local-example.yaml -A sudo obd cluster start c1 ``` +After you install OBD, you can run these commands as the root user to start a local single-node OceanBase cluster. +Before you run the commands, make sure that these conditions are met: + +- You have logged on as the root user. +- Ports `2881` and `2882` are available. +- Your server has at least 8 GB of memory. +- Your server has at least 2 CPU cores. + +> **NOTE:** If the preceding conditions are not met, see [OceanBase Deployer](https://github.com/oceanbase/obdeploy/blob/master/README.md). + > **NOTE:** We do not recommend that you use sys tenant to test. So please deploy clutser with optition -A, will create the test tenant during the bootstrap by using all available resources of the cluster. -> **NOTE:** Then you can create user. + +### Then create user for test. ```shell mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" diff --git a/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java index ddf50b745..13a0306bb 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseAggregate.java @@ -1,10 +1,9 @@ package sqlancer.oceanbase.ast; - public class OceanBaseAggregate implements OceanBaseExpression { - private OceanBaseExpression expr; - private OceanBaseAggregateFunction aggr; + private final OceanBaseExpression expr; + private final OceanBaseAggregateFunction aggr; public OceanBaseAggregate(OceanBaseExpression expr, OceanBaseAggregateFunction aggr) { this.expr = expr; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java index 3fe3bd32b..6edaa9357 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryComparisonOperation.java @@ -1,6 +1,5 @@ package sqlancer.oceanbase.ast; -import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; @@ -75,6 +74,7 @@ public OceanBaseConstant getExpectedValue(OceanBaseConstant leftVal, OceanBaseCo } } }; + private final String textRepresentation; public String getTextRepresentation() { @@ -96,7 +96,8 @@ public static BinaryComparisonOperator getRandom() { private final OceanBaseExpression right; private final BinaryComparisonOperator op; - public OceanBaseBinaryComparisonOperation(OceanBaseExpression left, OceanBaseExpression right, BinaryComparisonOperator op) { + public OceanBaseBinaryComparisonOperation(OceanBaseExpression left, OceanBaseExpression right, + BinaryComparisonOperator op) { this.left = left; this.right = right; this.op = op; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java index 0cb9ded11..2f9f69a87 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseBinaryLogicalOperation.java @@ -22,7 +22,7 @@ public OceanBaseConstant apply(OceanBaseConstant left, OceanBaseConstant right) return OceanBaseConstant.createFalse(); } } else if (right.isNull()) { - if (left.asBooleanNotNull()) { + if (left.asBooleanNotNull()) { return OceanBaseConstant.createNullConstant(); } else { return OceanBaseConstant.createFalse(); @@ -74,7 +74,8 @@ public static OceanBaseBinaryLogicalOperator getRandom() { } } - public OceanBaseBinaryLogicalOperation(OceanBaseExpression left, OceanBaseExpression right, OceanBaseBinaryLogicalOperator op) { + public OceanBaseBinaryLogicalOperation(OceanBaseExpression left, OceanBaseExpression right, + OceanBaseBinaryLogicalOperator op) { this.left = left; this.right = right; this.op = op; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java index c6ee0f350..ae1a2c001 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnName.java @@ -4,7 +4,7 @@ public class OceanBaseColumnName implements OceanBaseExpression { - private OceanBaseSchema.OceanBaseColumn column; + private final OceanBaseSchema.OceanBaseColumn column; public OceanBaseColumnName(OceanBaseSchema.OceanBaseColumn column) { this.column = column; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java index 64f903267..19753f813 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseColumnReference.java @@ -6,7 +6,7 @@ public class OceanBaseColumnReference implements OceanBaseExpression { private final OceanBaseColumn column; private final OceanBaseConstant value; - private boolean isRef = false; + private boolean isRef; public OceanBaseColumnReference(OceanBaseColumn column, OceanBaseConstant value) { this.column = column; @@ -30,12 +30,13 @@ public OceanBaseConstant getExpectedValue() { return value; } - public OceanBaseColumnReference setRef(boolean isRef){ - this.isRef = isRef; - return this; - } - public boolean getRef(){ - return isRef; + public OceanBaseColumnReference setRef(boolean isRef) { + this.isRef = isRef; + return this; + } + + public boolean getRef() { + return isRef; } } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java index ee78dd584..e9dd03ebe 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java @@ -14,7 +14,7 @@ public class OceanBaseComputableFunction implements OceanBaseExpression { public OceanBaseComputableFunction(OceanBaseFunction func, OceanBaseExpression... args) { this.func = func; - this.args = args; + this.args = args.clone(); } public OceanBaseFunction getFunction() { @@ -22,7 +22,7 @@ public OceanBaseFunction getFunction() { } public OceanBaseExpression[] getArguments() { - return args; + return args.clone(); } public enum OceanBaseFunction { @@ -77,7 +77,7 @@ public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] o return castToMostGeneralType(result, new OceanBaseExpression[] { origArgs[1], origArgs[2] }); } }, - + IFNULL(2, "IFNULL") { @Override @@ -87,7 +87,7 @@ public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] o result = args[1]; } else { result = args[0]; - }//args[0] and args[1] both null, if type is varchar, return null of varchar + } // args[0] and args[1] both null, if type is varchar, return null of varchar return castToMostGeneralType(result, origArgs); } @@ -111,11 +111,12 @@ public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpre final int nrArgs; private final boolean variadic; - private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] typeExpressions, BinaryOperator op) { + private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, + OceanBaseExpression[] typeExpressions, BinaryOperator op) { boolean containsNull = Stream.of(evaluatedArgs).anyMatch(arg -> arg.isNull()); if (containsNull) { - //IFNULL(GREATEST('aa',NULL), 0) -> '0' - // case1:c1 is float，value is NULL;select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' + // IFNULL(GREATEST('aa',NULL), 0) -> '0' + // case1:c1 is float，value is NULL;select COALESCE(GREATEST(NULL, concat(t1.c1)), 1) from t1;->'1' // select COALESCE(GREATEST(1, concat(t1.c1)), 1) from t1;->1 // select COALESCE(GREATEST('0', 1, concat(t1.c1)), 1) from t1;->1 // select COALESCE(GREATEST('0', concat(t1.c1)), 1) from t1;->'1' @@ -125,31 +126,35 @@ private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, Oc // select IFNULL(GREATEST("iffI|2&nBJLQQ", NULL, '0'), 1) from t0;->'1' OceanBaseDataType type; boolean allVarchar = true; - for (OceanBaseExpression expr : typeExpressions) { - if (expr instanceof OceanBaseColumnReference){ + for (OceanBaseExpression expr : typeExpressions) { + if (expr instanceof OceanBaseColumnReference) { type = ((OceanBaseColumnReference) expr).getColumn().getType(); - if(type == OceanBaseDataType.FLOAT) + if (type == OceanBaseDataType.FLOAT) { type = OceanBaseDataType.VARCHAR; - }else + } + } else { type = expr.getExpectedValue().getType(); - if(type != null && type.isNumeric()){ + } + if (type != null && type.isNumeric()) { allVarchar = false; break; } } - if (allVarchar) + if (allVarchar) { return OceanBaseConstant.createStringConstant("null"); - else + } else { return OceanBaseConstant.createNullConstant(); + } } OceanBaseConstant least = evaluatedArgs[1]; - /*select least(1,'H8*GPLuBjDj#Xem]W'); -> 0 - select least('1','H8*GPLuBjDj#Xem]W'); ->1 - select LEAST('000000000001', 'b', 1);->0*/ + /* + * select least(1,'H8*GPLuBjDj#Xem]W'); -> 0 select least('1','H8*GPLuBjDj#Xem]W'); ->1 select + * LEAST('000000000001', 'b', 1);->0 + */ OceanBaseDataType dataType = evaluatedArgs[0].getType(); boolean sameDataType = true; for (OceanBaseConstant arg : evaluatedArgs) { - if (arg.getType() != dataType){ + if (arg.getType() != dataType) { sameDataType = false; break; } @@ -157,19 +162,21 @@ private static OceanBaseConstant aggregate(OceanBaseConstant[] evaluatedArgs, Oc for (OceanBaseConstant arg : evaluatedArgs) { OceanBaseConstant left; OceanBaseConstant right; - if(sameDataType){ + if (sameDataType) { left = least; right = arg; - }else{ - //select GREATEST('1.47529e18', -1188315266);->1.47529e18 - if(least.getType() == OceanBaseDataType.VARCHAR) + } else { + // select GREATEST('1.47529e18', -1188315266);->1.47529e18 + if (least.getType() == OceanBaseDataType.VARCHAR) { left = least.castAsDouble(); - else + } else { left = least; - if(arg.getType() == OceanBaseDataType.VARCHAR) + } + if (arg.getType() == OceanBaseDataType.VARCHAR) { right = arg.castAsDouble(); - else + } else { right = arg; + } } least = op.apply(right, left); } @@ -192,7 +199,7 @@ public int getNrArgs() { return nrArgs; } - public abstract OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args); + public abstract OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args); public static OceanBaseFunction getRandomFunction() { return Randomly.fromOptions(values()); @@ -221,27 +228,29 @@ public OceanBaseConstant getExpectedValue() { return func.apply(constants, args); } - public static OceanBaseConstant castToMostGeneralType(OceanBaseConstant cons, OceanBaseExpression... typeExpressions) { + public static OceanBaseConstant castToMostGeneralType(OceanBaseConstant cons, + OceanBaseExpression... typeExpressions) { OceanBaseDataType type = getMostGeneralType(typeExpressions); if (cons.isNull()) { - if (type == OceanBaseDataType.FLOAT || type == OceanBaseDataType.VARCHAR) + if (type == OceanBaseDataType.FLOAT || type == OceanBaseDataType.VARCHAR) { return OceanBaseConstant.createStringConstant("null"); - else + } else { return cons; - }else{ - switch (type) { - case INT: - if (cons.isInt()) { + } + } else { + switch (type) { + case INT: + if (cons.isInt()) { + return cons; + } else { + return OceanBaseConstant.createIntConstant(cons.castAs(CastType.SIGNED).getInt()); + } + case VARCHAR: + return OceanBaseConstant.createStringConstant(cons.castAsString()); + default: return cons; - } else { - return OceanBaseConstant.createIntConstant(cons.castAs(CastType.SIGNED).getInt()); } - case VARCHAR: - return OceanBaseConstant.createStringConstant(cons.castAsString()); - default: - return cons; } - } } public static OceanBaseDataType getMostGeneralType(OceanBaseExpression... expressions) { @@ -250,15 +259,17 @@ public static OceanBaseDataType getMostGeneralType(OceanBaseExpression... expres OceanBaseDataType exprType; if (expr instanceof OceanBaseColumnReference) { exprType = ((OceanBaseColumnReference) expr).getColumn().getType(); - if(((OceanBaseColumnReference) expr).getColumn().isZeroFill()) + if (((OceanBaseColumnReference) expr).getColumn().isZeroFill()) { exprType = OceanBaseDataType.VARCHAR; + } } else { exprType = expr.getExpectedValue().getType(); } if (type == null) { type = exprType; - if (exprType == OceanBaseDataType.FLOAT) + if (exprType == OceanBaseDataType.FLOAT) { type = OceanBaseDataType.VARCHAR; + } } else if (exprType == OceanBaseDataType.VARCHAR || exprType == OceanBaseDataType.FLOAT) { type = OceanBaseDataType.VARCHAR; } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java index d99447fb6..174defdb8 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -1,7 +1,7 @@ package sqlancer.oceanbase.ast; -import java.math.BigInteger; import java.math.BigDecimal; +import java.math.BigInteger; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -17,7 +17,7 @@ public boolean isInt() { public boolean isNull() { return false; } - + public boolean isDouble() { return false; } @@ -85,7 +85,7 @@ public OceanBaseDoubleConstant(double val) { public String getTextRepresentation() { return String.valueOf(val); } - + @Override public double getDouble() { return this.val; @@ -105,22 +105,24 @@ public boolean asBooleanNotNull() { public OceanBaseConstant castAs(CastType type) { if (type == CastType.SIGNED) { long value = new Double(val).longValue(); - if (val - value >=0.5) - value = value +1; + if (val - value >= 0.5) { + value = value + 1; + } return new OceanBaseIntConstant(value, true); } else if (type == CastType.UNSIGNED) { long value = new Double(val).longValue(); - if (val - value >=0.5) - value = value +1; + if (val - value >= 0.5) { + value = value + 1; + } return new OceanBaseIntConstant(value, false); } else { throw new AssertionError(); } - } + } @Override public String castAsString() { - return String.valueOf(new BigDecimal(val));//select IFNULL(1.713591018E9, '11') -> 1713591018 + return String.valueOf(new BigDecimal(val)); // select IFNULL(1.713591018E9, '11') -> 1713591018 } @Override @@ -132,31 +134,32 @@ public boolean isDouble() { protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { if (rightVal.isNull()) { return OceanBaseConstant.createNullConstant(); - } else if (rightVal instanceof OceanBaseIntConstant){ + } else if (rightVal instanceof OceanBaseIntConstant) { return OceanBaseConstant.createBoolean(val < rightVal.getInt()); - } else if (rightVal instanceof OceanBaseDoubleConstant){ + } else if (rightVal instanceof OceanBaseDoubleConstant) { return OceanBaseConstant.createBoolean(val < rightVal.getDouble()); - } else if (rightVal instanceof OceanBaseTextConstant){ + } else if (rightVal instanceof OceanBaseTextConstant) { return isLessThan(rightVal.castAsDouble()); - } else{ + } else { throw new AssertionError(rightVal); - } + } } @Override public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { if (rightVal.isNull()) { return OceanBaseConstant.createNullConstant(); - } else if (rightVal instanceof OceanBaseIntConstant){ + } else if (rightVal instanceof OceanBaseIntConstant) { return OceanBaseConstant.createBoolean(val == rightVal.getInt()); - } else if (rightVal instanceof OceanBaseDoubleConstant){ + } else if (rightVal instanceof OceanBaseDoubleConstant) { return OceanBaseConstant.createBoolean(val == rightVal.getDouble()); - } else if (rightVal instanceof OceanBaseTextConstant){ + } else if (rightVal instanceof OceanBaseTextConstant) { return isEquals(rightVal.castAsDouble()); } else { throw new AssertionError(rightVal); } } + @Override public OceanBaseDataType getType() { return OceanBaseDataType.DOUBLE; @@ -182,37 +185,32 @@ private void checkIfSmallFloatingPointText() { throw new IgnoreMeException(); } } + @Override public boolean isNull() { - if(value.equalsIgnoreCase("NULL")) - return true; - else - return false; + return value.equalsIgnoreCase("NULL"); } @Override - public boolean isEmpty(){ - //"" " " - if (value.length() ==0 ){ + public boolean isEmpty() { + // "" " " + if (value.length() == 0) { return true; - }else{ - for(int i =0;i< value.length(); i++){ - String sub =value.substring(i, i+1); - if(!sub.equals(" ")) - return false; + } else { + for (int i = 0; i < value.length(); i++) { + String sub = value.substring(i, i + 1); + if (!sub.equals(" ")) { + return false; + } } return true; - } + } } @Override public boolean asBooleanNotNull() { for (int i = value.length(); i >= 1; i--) { try { - char currentChar = value.charAt(i-1); - int currentVal= Integer.valueOf(currentChar); - if (currentVal < 48||currentVal > 57) - continue; String substring = value.substring(0, i); Double val = Double.valueOf(substring); return val != 0 && !Double.isNaN(val); @@ -247,10 +245,11 @@ public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { } else if (rightVal instanceof OceanBaseDoubleConstant) { return castAsDouble().isEquals(rightVal); } else if (rightVal.isString()) { - if (isEmpty() && rightVal.isEmpty()) + if (isEmpty() && rightVal.isEmpty()) { return OceanBaseConstant.createBoolean(true); - else + } else { return OceanBaseConstant.createBoolean(value.equalsIgnoreCase(rightVal.getString())); + } } else { throw new AssertionError(rightVal); } @@ -268,8 +267,9 @@ public boolean isString() { @Override public OceanBaseConstant castAs(CastType type) { - if(isNull()) + if (isNull()) { return OceanBaseConstant.createNullConstant(); + } if (type == CastType.SIGNED || type == CastType.UNSIGNED) { String value = this.value; while (value.startsWith(" ") || value.startsWith("\t") || value.startsWith("\n")) { @@ -280,11 +280,6 @@ public OceanBaseConstant castAs(CastType type) { } for (int i = value.length(); i >= 1; i--) { try { - //select CAST("꯵z)" AS SIGNED); ->0 - char currentChar = value.charAt(i-1); - int currentVal= Integer.valueOf(currentChar); - if (currentVal < 48||currentVal > 57) - throw new IgnoreMeException(); String substring = value.substring(0, i); long val = Long.parseLong(substring); return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); @@ -309,11 +304,6 @@ public OceanBaseConstant castAsDouble() { } for (int i = value.length(); i >= 1; i--) { try { - char currentChar = value.charAt(i-1); - int currentVal= Integer.valueOf(currentChar); - if (currentVal < 48||currentVal > 57) - //ignore special char - throw new IgnoreMeException(); String substring = value.substring(0, i); double val = Double.parseDouble(substring); return OceanBaseConstant.createDoubleConstant(val); @@ -414,7 +404,7 @@ public OceanBaseConstant isEquals(OceanBaseConstant rightVal) { throw new IgnoreMeException(); } return isEquals(rightVal.castAs(CastType.SIGNED)); - } else if (rightVal instanceof OceanBaseDoubleConstant){ + } else if (rightVal instanceof OceanBaseDoubleConstant) { return OceanBaseConstant.createBoolean(value == rightVal.getDouble()); } else { throw new AssertionError(rightVal); @@ -440,7 +430,7 @@ public String castAsString() { return Long.toUnsignedString(value); } } - + @Override public OceanBaseConstant castAsDouble() { return this; @@ -481,7 +471,7 @@ protected OceanBaseConstant isLessThan(OceanBaseConstant rightVal) { throw new IgnoreMeException(); } return isLessThan(rightVal.castAs(isSigned ? CastType.SIGNED : CastType.UNSIGNED)); - } else if (rightVal instanceof OceanBaseDoubleConstant){ + } else if (rightVal instanceof OceanBaseDoubleConstant) { return OceanBaseConstant.createBoolean(value < rightVal.getDouble()); } else { throw new AssertionError(rightVal); diff --git a/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java index e5c482a56..c4caec195 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseInOperation.java @@ -2,8 +2,6 @@ import java.util.List; -import sqlancer.IgnoreMeException; - public class OceanBaseInOperation implements OceanBaseExpression { private final OceanBaseExpression expr; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java index 642553173..a87ff5c27 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java @@ -1,7 +1,7 @@ package sqlancer.oceanbase.ast; -import java.util.Collections; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import sqlancer.common.ast.SelectBase; @@ -28,12 +28,15 @@ public SelectType getFromOptions() { public void setFromOptions(SelectType fromOptions) { this.fromOptions = fromOptions; } - public void setGroupByClause(List groupBys){ + + public void setGroupByClause(List groupBys) { this.groupBys = groupBys; } - public List getGroupByClause(){ + + public List getGroupByClause() { return this.groupBys; } + public void setModifiers(List modifiers) { this.modifiers = modifiers; } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseText.java b/src/sqlancer/oceanbase/ast/OceanBaseText.java index 566effde4..f64cba17c 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseText.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseText.java @@ -1,11 +1,10 @@ package sqlancer.oceanbase.ast; - public class OceanBaseText implements OceanBaseExpression { - private OceanBaseExpression expr; - private String text; - private boolean prefix; + private final OceanBaseExpression expr; + private final String text; + private final boolean prefix; public OceanBaseText(OceanBaseExpression expr, String text, boolean prefix) { this.expr = expr; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java index c598e3bd0..e659f75be 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseUnaryPrefixOperation.java @@ -65,11 +65,12 @@ public OceanBaseUnaryPrefixOperation(OceanBaseExpression expr, OceanBaseUnaryPre @Override public OceanBaseConstant getExpectedValue() { OceanBaseConstant subExprVal = expr.getExpectedValue(); - if (op == OceanBaseUnaryPrefixOperator.PLUS){ - if (subExprVal.isNull() && subExprVal.getType() == null) + if (op == OceanBaseUnaryPrefixOperator.PLUS) { + if (subExprVal.isNull() && subExprVal.getType() == null) { return OceanBaseConstant.createNullConstant(); - else + } else { return subExprVal; + } } if (subExprVal.isNull()) { return OceanBaseConstant.createNullConstant(); @@ -82,9 +83,11 @@ public OceanBaseConstant getExpectedValue() { public OperatorKind getOperatorKind() { return OperatorKind.PREFIX; } + public OceanBaseExpression getExpr() { return expr; } + public OceanBaseUnaryPrefixOperator getOp() { return op; } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java index b3f7de7bb..e67256448 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; @@ -29,6 +28,7 @@ public static SQLQueryAdapter create(OceanBaseGlobalState globalState) { private enum Action { COMPRESSION; + private String[] potentialErrors; Action(String... couldCauseErrors) { @@ -59,6 +59,8 @@ private SQLQueryAdapter create() { sb.append(Randomly.fromOptions("ZLIB_1.0", "LZ4_1.0", "NONE")); sb.append("'"); break; + default: + break; } } for (Action a : selectedActions) { diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index de76917e8..f2a6b2731 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -15,6 +15,7 @@ public class OceanBaseDeleteGenerator { private final StringBuilder sb = new StringBuilder(); private final OceanBaseGlobalState globalState; private final Randomly r; + public OceanBaseDeleteGenerator(OceanBaseGlobalState globalState) { this.globalState = globalState; this.r = globalState.getRandomly(); @@ -26,11 +27,12 @@ public static SQLQueryAdapter delete(OceanBaseGlobalState globalState) { private SQLQueryAdapter generate() { OceanBaseTable randomTable = globalState.getSchema().getRandomTable(); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(randomTable.getColumns()); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState) + .setColumns(randomTable.getColumns()); ExpectedErrors errors = new ExpectedErrors(); sb.append("DELETE"); if (Randomly.getBoolean()) { - sb.append(" /*+parallel("+r.getLong(0, 10)+") enable_parallel_dml*/ "); + sb.append(" /*+parallel(" + r.getLong(0, 10) + ") enable_parallel_dml*/ "); } sb.append(" FROM "); sb.append(randomTable.getName()); @@ -39,12 +41,9 @@ private SQLQueryAdapter generate() { sb.append(OceanBaseVisitor.asString(gen.generateExpression())); OceanBaseErrors.addExpressionErrors(errors); } - errors.addAll(Arrays.asList("doesn't have this option", - "Truncated incorrect DOUBLE value", - "Truncated incorrect INTEGER value", - "Truncated incorrect DECIMAL value", - "Data truncated for functional index", - "Incorrect value", "Out of range value for column", + errors.addAll(Arrays.asList("doesn't have this option", "Truncated incorrect DOUBLE value", + "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", + "Data truncated for functional index", "Incorrect value", "Out of range value for column", "Data truncation: %s value is out of range in '%s'")); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java index 5e7c06fb2..58a17d3ac 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDropIndex.java @@ -1,7 +1,6 @@ package sqlancer.oceanbase.gen; import sqlancer.IgnoreMeException; -import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.oceanbase.OceanBaseGlobalState; @@ -22,9 +21,8 @@ public static SQLQueryAdapter generate(OceanBaseGlobalState globalState) { sb.append(table.getRandomIndex().getIndexName()); sb.append(" ON "); sb.append(table.getName()); - return new SQLQueryAdapter(sb.toString(), - ExpectedErrors.from("LOCK=NONE is not supported", "ALGORITHM=INPLACE is not supported", - "Data truncation", "Data truncated for functional index")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("LOCK=NONE is not supported", + "ALGORITHM=INPLACE is not supported", "Data truncation", "Data truncated for functional index")); } } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java index 279223d73..f4f573698 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java @@ -1,11 +1,9 @@ package sqlancer.oceanbase.gen; +import java.sql.Connection; import java.util.ArrayList; import java.util.List; -import java.sql.Connection; - -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.oceanbase.OceanBaseGlobalState; @@ -25,8 +23,6 @@ import sqlancer.oceanbase.ast.OceanBaseExists; import sqlancer.oceanbase.ast.OceanBaseExpression; import sqlancer.oceanbase.ast.OceanBaseInOperation; -import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; -import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; import sqlancer.oceanbase.ast.OceanBaseStringExpression; import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; @@ -36,27 +32,23 @@ public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator OceanBaseColumns = new ArrayList<>(); public OceanBaseExpressionGenerator(OceanBaseGlobalState state) { this.state = state; } + public OceanBaseExpressionGenerator setCon(Connection con) { return this; } + public OceanBaseExpressionGenerator setState(OceanBaseGlobalState state) { this.state = state; return this; } public OceanBaseExpressionGenerator setOceanBaseColumns(List columns) { - this.OceanBaseColumns = columns; return this; } - public OceanBaseExpressionGenerator(Randomly r) { - this.r = r; - } public OceanBaseExpressionGenerator setRowVal(OceanBaseRowValue rowVal) { this.rowVal = rowVal; @@ -64,7 +56,7 @@ public OceanBaseExpressionGenerator setRowVal(OceanBaseRowValue rowVal) { } private enum Actions { - COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, + COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, EXISTS; } @@ -95,7 +87,8 @@ public OceanBaseExpression generateExpression(int depth) { return new OceanBaseBinaryComparisonOperation(generateExpression(depth + 1), generateExpression(depth + 1), BinaryComparisonOperator.getRandom()); case CAST: - return new OceanBaseCastOperation(generateExpression(depth + 1), OceanBaseCastOperation.CastType.getRandom()); + return new OceanBaseCastOperation(generateExpression(depth + 1), + OceanBaseCastOperation.CastType.getRandom()); case IN_OPERATION: OceanBaseExpression expr = generateExpression(depth + 1); List rightList = new ArrayList<>(); @@ -114,7 +107,8 @@ private OceanBaseExpression getExists() { if (Randomly.getBoolean()) { return new OceanBaseExists(new OceanBaseStringExpression("SELECT 1", OceanBaseConstant.createTrue())); } else { - return new OceanBaseExists(new OceanBaseStringExpression("SELECT 1 from dual wHERE FALSE", OceanBaseConstant.createFalse())); + return new OceanBaseExists( + new OceanBaseStringExpression("SELECT 1 from dual wHERE FALSE", OceanBaseConstant.createFalse())); } } @@ -133,11 +127,12 @@ private OceanBaseExpression getComputableFunction(int depth) { private enum ConstantType { INT, NULL, STRING, DOUBLE; + public static ConstantType[] valuesPQS() { return new ConstantType[] { INT, NULL, STRING }; } } - + @Override public OceanBaseExpression generateConstant() { ConstantType[] values; @@ -146,6 +141,7 @@ public OceanBaseExpression generateConstant() { } else { values = ConstantType.values(); } + OceanBaseConstant constant; switch (Randomly.fromOptions(values)) { case INT: return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); @@ -153,11 +149,12 @@ public OceanBaseExpression generateConstant() { return OceanBaseConstant.createNullConstant(); case STRING: String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); - OceanBaseConstant createStringConstant = OceanBaseConstant.createStringConstant(string); - return createStringConstant; + constant = OceanBaseConstant.createStringConstant(string); + return constant; case DOUBLE: double val = state.getRandomly().getDouble(); - return new OceanBaseDoubleConstant(val); + constant = new OceanBaseDoubleConstant(val); + return constant; default: throw new AssertionError(); } @@ -176,6 +173,7 @@ public OceanBaseExpression generateColumn() { } public OceanBaseExpression generateConstant(OceanBaseColumn col) { + OceanBaseConstant constant; switch (col.getType().name()) { case "INT": return OceanBaseConstant.createIntConstant((int) state.getRandomly().getInteger()); @@ -183,22 +181,24 @@ public OceanBaseExpression generateConstant(OceanBaseColumn col) { return OceanBaseConstant.createNullConstant(); case "VARCHAR": String string = state.getRandomly().getString().replace("\\", "").replace("\n", "").replace("\t", ""); - OceanBaseConstant createStringConstant = OceanBaseConstant.createStringConstant(string); - return createStringConstant; + constant = OceanBaseConstant.createStringConstant(string); + return constant; case "DOUBLE": double val = state.getRandomly().getDouble(); - return new OceanBaseDoubleConstant(val); + constant = new OceanBaseDoubleConstant(val); + return constant; case "FLOAT": - val = state.getRandomly().getDouble(); - return new OceanBaseDoubleConstant(val); + val = state.getRandomly().getDouble(); + constant = new OceanBaseDoubleConstant(val); + return constant; case "DECIMAL": - val = state.getRandomly().getDouble(); + val = state.getRandomly().getDouble(); return new OceanBaseDoubleConstant(val); default: throw new AssertionError(); } } - + @Override public OceanBaseExpression negatePredicate(OceanBaseExpression predicate) { return new OceanBaseUnaryPrefixOperation(predicate, OceanBaseUnaryPrefixOperator.NOT); @@ -206,7 +206,7 @@ public OceanBaseExpression negatePredicate(OceanBaseExpression predicate) { @Override public OceanBaseExpression isNull(OceanBaseExpression expr) { - return new OceanBaseUnaryPostfixOperation(expr, OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + return new OceanBaseUnaryPostfixOperation(expr, OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, + false); } } - diff --git a/src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java new file mode 100644 index 000000000..8c5b66841 --- /dev/null +++ b/src/sqlancer/oceanbase/gen/OceanBaseHintGenerator.java @@ -0,0 +1,125 @@ +package sqlancer.oceanbase.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseStringExpression; + +public class OceanBaseHintGenerator { + private final OceanBaseSelect select; + private final List tables; + private final StringBuilder sb = new StringBuilder(); + private final Randomly r = new Randomly(); + + enum IndexHint { + PDML, NO_PRED_DEDUCE, MERGE_JOIN, HASH_JOIN, NL_JOIN, BNL_JOIN, NO_MERGE_JOIN, NO_HASH_JOIN, NO_NL_JOIN, + NO_BNL_JOIN, HASH_AGG, NL_MATERIALIZATION, LATE_MATERIALIZATION, USE_INDEX, TOPK, LEADING, ORDERED, NO_REWRITE; + } + + public OceanBaseHintGenerator(OceanBaseSelect select, List tables) { + this.select = select; + this.tables = tables; + } + + public static void generateHints(OceanBaseSelect select, List tables) { + new OceanBaseHintGenerator(select, tables).generate(); + + } + + private void generate() { + switch (Randomly.fromOptions(IndexHint.values())) { + case PDML: + sb.append(" parallel(" + r.getInteger(0, 10) + "),enable_parallel_dml "); + break; + case NO_PRED_DEDUCE: + sb.append("NO_PRED_DEDUCE"); + break; + case MERGE_JOIN: + tablesHint("USE_MERGE "); + break; + case HASH_JOIN: + tablesHint("USE_HASH "); + break; + case NL_JOIN: + tablesHint("USE_NL "); + break; + case BNL_JOIN: + tablesHint("USE_BNL "); + break; + case NO_MERGE_JOIN: + sb.append(" NO_USE_MERGE "); + break; + case NO_HASH_JOIN: + sb.append(" NO_USE_HASH "); + break; + case NO_NL_JOIN: + sb.append(" NO_USE_NL "); + break; + case NO_BNL_JOIN: + sb.append(" NO_USE_BNL "); + break; + case HASH_AGG: + sb.append("USE_HASH_AGGREGATION "); + break; + case NL_MATERIALIZATION: + sb.append("USE_NL_MATERIALIZATION "); + break; + case LATE_MATERIALIZATION: + sb.append("USE_LATE_MATERIALIZATION "); + break; + case USE_INDEX: + indexesHint("INDEX_HINT "); + break; + case TOPK: + sb.append("TOPK (50 50) "); + break; + case LEADING: + tablesHint(" LEADING "); + break; + case ORDERED: + sb.append("ORDERED "); + break; + case NO_REWRITE: + sb.append("NO_REWRITE "); + break; + default: + throw new AssertionError(); + } + + select.setHint(new OceanBaseStringExpression(sb.toString(), + new OceanBaseConstant.OceanBaseTextConstant(sb.toString()))); + } + + private void indexesHint(String string) { + sb.append(string); + sb.append("("); + OceanBaseTable table = Randomly.fromList(tables); + List allIndexes = table.getIndexes(); + if (allIndexes.isEmpty()) { + throw new IgnoreMeException(); + } + List indexSubset = Randomly.nonEmptySubset(allIndexes); + sb.append(table.getName()); + sb.append(", "); + sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); + sb.append(")"); + } + + private void tablesHint(String string) { + sb.append(string); + sb.append("("); + appendTables(); + sb.append(")"); + } + + private void appendTables() { + List tableSubset = Randomly.nonEmptySubset(tables); + sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + } +} diff --git a/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java index aca0511ce..79d206f70 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseInsertGenerator.java @@ -20,7 +20,7 @@ public class OceanBaseInsertGenerator { private final ExpectedErrors errors = new ExpectedErrors(); private final OceanBaseGlobalState globalState; private final Randomly r; - private int type = 0; + private int type; public OceanBaseInsertGenerator(OceanBaseGlobalState globalState) { this.globalState = globalState; @@ -46,7 +46,7 @@ private SQLQueryAdapter generateReplace() { private SQLQueryAdapter generateInsert() { sb.append("INSERT"); if (Randomly.getBoolean()) { - sb.append(" /*+parallel("+r.getLong(0, 10)+") enable_parallel_dml*/ "); + sb.append(" /*+parallel(" + r.getLong(0, 10) + ") enable_parallel_dml*/ "); } if (Randomly.getBoolean()) { sb.append(" "); @@ -62,7 +62,7 @@ private SQLQueryAdapter generateInto() { sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); sb.append(") "); sb.append("VALUES"); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); int nrRows; if (Randomly.getBoolean()) { nrRows = 1; @@ -84,12 +84,10 @@ private SQLQueryAdapter generateInto() { } sb.append(")"); } - if(Randomly.getBoolean() && type == 0){ - + if (Randomly.getBoolean() && type == 0) { List upcolumns = table.getRandomNonEmptyColumnSubset(); - if (upcolumns.size() > 0 ){ + if (!upcolumns.isEmpty()) { sb.append(" ON DUPLICATE KEY UPDATE "); - sb.append(upcolumns.get(0).getName()); sb.append("="); sb.append(gen.generateConstant(upcolumns.get(0))); @@ -106,5 +104,4 @@ private SQLQueryAdapter generateInto() { errors.add("Duplicated primary key"); return new SQLQueryAdapter(sb.toString(), errors); } - } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java index bf4f679a9..2a193c1ae 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseTableGenerator.java @@ -6,7 +6,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -23,8 +22,6 @@ public class OceanBaseTableGenerator { private final String tableName; private final Randomly r; private int columnId; - private boolean tableHasNullableColumn; - private int keysSpecified; private final List columns = new ArrayList<>(); private final OceanBaseSchema schema; private final OceanBaseGlobalState globalState; @@ -109,8 +106,8 @@ private void appendPartitionOptions() { } private enum TableOptions { - BS,BLOOM,AUTO_INCREMENT; - + BS, BLOOM, AUTO_INCREMENT; + public static List getRandomTableOptions() { List options; if (Randomly.getBooleanWithSmallProbability()) { @@ -134,25 +131,27 @@ private void appendTableOptions() { sb.append(", "); } switch (o) { - case AUTO_INCREMENT: - sb.append("AUTO_INCREMENT = " + r.getPositiveInteger()); - break; - case BLOOM: - sb.append("USE_BLOOM_FILTER = "); - if(Randomly.getBoolean()) - sb.append(" FALSE "); - else - sb.append(" true "); - break; - case BS: - sb.append(" BLOCK_SIZE = "); - if(Randomly.getBoolean()) - sb.append(" 16384 "); - else - sb.append(" 32768 "); - break; - default: - throw new AssertionError(o); + case AUTO_INCREMENT: + sb.append("AUTO_INCREMENT = " + r.getPositiveInteger()); + break; + case BLOOM: + sb.append("USE_BLOOM_FILTER = "); + if (Randomly.getBoolean()) { + sb.append(" FALSE "); + } else { + sb.append(" true "); + } + break; + case BS: + sb.append(" BLOCK_SIZE = "); + if (Randomly.getBoolean()) { + sb.append(" 16384 "); + } else { + sb.append(" 32768 "); + } + break; + default: + throw new AssertionError(o); } } } @@ -171,7 +170,7 @@ private enum ColumnOptions { private void appendColumnDefinition() { sb.append(" "); - + OceanBaseDataType randomType = OceanBaseDataType.getRandom(globalState); boolean isTextType = randomType == OceanBaseDataType.VARCHAR; appendTypeString(randomType); @@ -180,9 +179,6 @@ private void appendColumnDefinition() { boolean columnHasPrimaryKey = false; List columnOptions = Randomly.subset(ColumnOptions.values()); - if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { - tableHasNullableColumn = true; - } if (isTextType) { columnOptions.remove(ColumnOptions.PRIMARY_KEY); columnOptions.remove(ColumnOptions.UNIQUE); @@ -196,7 +192,6 @@ private void appendColumnDefinition() { if (Randomly.getBoolean()) { sb.append("NULL"); } - tableHasNullableColumn = true; isNull = true; } else { sb.append("NOT NULL"); @@ -204,13 +199,11 @@ private void appendColumnDefinition() { break; case UNIQUE: sb.append("UNIQUE"); - keysSpecified++; if (Randomly.getBoolean()) { sb.append(" KEY"); } break; case COMMENT: - // TODO: generate randomly sb.append(String.format("COMMENT '%s' ", "asdf")); break; case PRIMARY_KEY: @@ -238,7 +231,7 @@ private void appendTypeString(OceanBaseDataType randomType) { sb.append(Randomly.fromOptions("TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT")); if (Randomly.getBoolean()) { sb.append("("); - sb.append(Randomly.getNotCachedInteger(0, 255)); + sb.append(Randomly.getNotCachedInteger(0, 255)); sb.append(")"); } break; diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index ff20328d5..f89434498 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -1,28 +1,37 @@ package sqlancer.oceanbase.gen; +import java.util.List; + +import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.Randomly; -import sqlancer.oceanbase.*; +import sqlancer.oceanbase.OceanBaseErrors; +import sqlancer.oceanbase.OceanBaseGlobalState; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseVisitor; +public class OceanBaseUpdateGenerator { -import java.sql.SQLException; -import java.util.HashSet; -import java.util.List; -import java.util.Set; + private final StringBuilder sb = new StringBuilder(); + private final OceanBaseGlobalState globalState; + private final Randomly r; -public class OceanBaseUpdateGenerator { - private OceanBaseUpdateGenerator() { + public OceanBaseUpdateGenerator(OceanBaseGlobalState globalState) { + this.globalState = globalState; + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter update(OceanBaseGlobalState globalState) { + return new OceanBaseUpdateGenerator(globalState).generate(); } - public static SQLQueryAdapter getQuery(OceanBaseGlobalState globalState) throws SQLException { - Randomly r = globalState.getRandomly(); + private SQLQueryAdapter generate() { ExpectedErrors errors = new ExpectedErrors(); OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); - StringBuilder sb = new StringBuilder("UPDATE "); + sb.append("UPDATE "); if (Randomly.getBoolean()) { - sb.append(" /*+parallel("+r.getInteger(0, 10)+") enable_parallel_dml*/ "); + sb.append(" /*+parallel(" + r.getInteger(0, 10) + ") enable_parallel_dml*/ "); } sb.append(table.getName()); sb.append(" SET "); @@ -44,7 +53,7 @@ public static SQLQueryAdapter getQuery(OceanBaseGlobalState globalState) throws sb.append(" WHERE "); OceanBaseErrors.addExpressionErrors(errors); sb.append(OceanBaseVisitor.asString(gen.generateExpression())); - errors.add("Data Too Long"); + errors.add("Data Too Long"); } errors.add("Duplicated primary key"); OceanBaseErrors.addInsertErrors(errors); diff --git a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java index 09daea34a..b7812a7ff 100644 --- a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java +++ b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java @@ -1,9 +1,7 @@ package sqlancer.oceanbase.gen.datadef; -import java.util.List; -import java.util.Set; -import java.util.HashSet; import java.util.ArrayList; +import java.util.List; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; @@ -14,18 +12,13 @@ import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.OceanBaseVisitor; -import sqlancer.oceanbase.ast.OceanBaseExpression; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; public class OceanBaseIndexGenerator { private final Randomly r; private StringBuilder sb = new StringBuilder(); - private boolean columnIsPrimaryKey; - private boolean containsInPlace; private OceanBaseSchema schema; - private List columns = new ArrayList<>(); + private final List columns = new ArrayList<>(); private final OceanBaseGlobalState globalState; public OceanBaseIndexGenerator(OceanBaseSchema schema, Randomly r, OceanBaseGlobalState globalState) { @@ -47,17 +40,13 @@ public SQLQueryAdapter create() { indexType(); sb.append(" ON "); OceanBaseTable table = schema.getRandomTable(); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(table.getName()); sb.append("("); List randomColumn = table.getRandomNonEmptyColumnSubset(); int i = 0; for (OceanBaseColumn c : randomColumn) { if (i++ != 0) { - sb.append(", "); - } - if (c.isPrimaryKey()) { - columnIsPrimaryKey = true; + sb.append(", "); } c.isPartioned = true; columns.add(c); @@ -96,6 +85,7 @@ public SQLQueryAdapter create() { private enum PartitionOptions { HASH } + private void appendPartitionOptions() { if (Randomly.getBoolean()) { return; @@ -103,11 +93,11 @@ private void appendPartitionOptions() { OceanBaseColumn colIndex = Randomly.fromList(columns); - if(colIndex.isPartioned == false){ + if (!colIndex.isPartioned) { return; } - if(colIndex.getType() == OceanBaseDataType.VARCHAR){ + if (colIndex.getType() == OceanBaseDataType.VARCHAR) { sb.append(" PARTITION BY"); sb.append(" KEY"); sb.append(" ("); @@ -116,8 +106,7 @@ private void appendPartitionOptions() { sb.append(")"); sb.append(" partitions "); sb.append(r.getInteger(1, 20)); - } - else if(OceanBaseDataType.INT == colIndex.getType()){ + } else if (OceanBaseDataType.INT == colIndex.getType()) { sb.append(" PARTITION BY"); sb.append(" HASH("); String name = colIndex.getName(); @@ -125,8 +114,7 @@ else if(OceanBaseDataType.INT == colIndex.getType()){ sb.append(") "); sb.append(" partitions "); sb.append(r.getInteger(1, 20)); - } - else{ + } else { return; } } diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java index 85748870e..900ac8caa 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java @@ -1,5 +1,11 @@ package sqlancer.oceanbase.oracle; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; @@ -8,37 +14,34 @@ import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; import sqlancer.oceanbase.OceanBaseVisitor; -import sqlancer.oceanbase.ast.*; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; +import sqlancer.oceanbase.ast.OceanBaseColumnName; import sqlancer.oceanbase.ast.OceanBaseComputableFunction; import sqlancer.oceanbase.ast.OceanBaseComputableFunction.OceanBaseFunction; import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseText; +import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; -import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; -import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; - -import java.sql.Connection; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import java.util.Random; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { +public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { // SELECT COUNT(*) FROM t0 WHERE ; // SELECT SUM(count) FROM (SELECT IS TRUE as count FROM t0); // SELECT (SELECT COUNT(*) FROM t0 WHERE c0 IS NOT 0) = (SELECT COUNT(*) FROM // (SELECT c0 is NOT 0 FROM t0)); - private OceanBaseSchema s; - private Randomly r; + private final OceanBaseSchema s; private String firstQueryString; - private String secondQueryString; private static final int NOT_FOUND = -1; + private enum Option { + TRUE, FALSE_NULL, NOT_NOT_TRUE, NOT_FALSE_NOT_NULL, IF, IFNULL, COALESCE + }; + public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { super(globalState); this.s = globalState.getSchema(); @@ -56,14 +59,15 @@ public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { errors.add("digit expected after"); } + @Override public void check() throws SQLException { OceanBaseSchema.OceanBaseTable randomTable = s.getRandomTable(); List columns = randomTable.getColumns(); OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state).setColumns(columns); OceanBaseExpression randomWhereCondition = gen.generateExpression(); List groupBys = Collections.emptyList(); // getRandomExpressions(columns); - List tableList = (Arrays.asList(randomTable)).stream().map(t -> new OceanBaseTableReference(t)) - .collect(Collectors.toList()); + List tableList = Arrays.asList(randomTable).stream() + .map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList()); int firstCount = getFirstQueryCount(tableList, randomWhereCondition, groupBys); int secondCount = getSecondQuery(tableList, randomWhereCondition, groupBys); if (firstCount != secondCount && firstCount != NOT_FOUND && secondCount != NOT_FOUND) { @@ -79,7 +83,7 @@ public void check() throws SQLException { } private int getSecondQuery(List tableList, OceanBaseExpression randomWhereCondition, - List groupBys) throws SQLException { + List groupBys) throws SQLException { OceanBaseSelect select = new OceanBaseSelect(); select.setGroupByClause(groupBys); OceanBaseExpression expr = getTrueExpr(randomWhereCondition); @@ -117,8 +121,8 @@ private int getFirstQueryCount(List tableList, OceanBaseExp select.setGroupByClause(groupBys); // SELECT COUNT(t1.c3) FROM t1 WHERE (- (t1.c2)); // SELECT SUM(count) FROM (SELECT ((- (t1.c2)) IS TRUE) as count FROM t1);; - OceanBaseAggregate aggr = new OceanBaseAggregate( - new OceanBaseColumnName(new OceanBaseSchema.OceanBaseColumn("*", OceanBaseSchema.OceanBaseDataType.INT, false, 0, false)), + OceanBaseAggregate aggr = new OceanBaseAggregate(new OceanBaseColumnName( + new OceanBaseSchema.OceanBaseColumn("*", OceanBaseSchema.OceanBaseDataType.INT, false, 0, false)), OceanBaseAggregate.OceanBaseAggregateFunction.COUNT); select.setFetchColumns(Arrays.asList(aggr)); select.setFromList(tableList); @@ -145,60 +149,76 @@ private int getFirstQueryCount(List tableList, OceanBaseExp rs.close(); return firstCount; } - - private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition){ - int i = new Random().nextInt(8); - OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE,false); - - OceanBaseUnaryPostfixOperation isNotTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE,true); - - OceanBaseUnaryPostfixOperation isFalse= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE,false); - - OceanBaseUnaryPostfixOperation isNotFalse= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE,true); - - OceanBaseUnaryPostfixOperation isNULL= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL,false); - - OceanBaseUnaryPostfixOperation isNotNULL= new OceanBaseUnaryPostfixOperation(randomWhereCondition,OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL,true); - - OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); - switch(i){ - case 0: - expr = isTrue; - break; - case 1: - expr = new OceanBaseUnaryPrefixOperation(new OceanBaseBinaryLogicalOperation(isFalse, isNULL, OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case 2: - expr = new OceanBaseUnaryPrefixOperation(new OceanBaseUnaryPrefixOperation(isTrue, OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case 3: - expr = new OceanBaseUnaryPrefixOperation(isNotTrue, OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case 4: - expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); - break; - case 5: - OceanBaseExpression[] args = new OceanBaseExpression[3]; - args[0] = OceanBaseConstant.createIntConstant(1); - args[1] = isTrue; - args[2] = OceanBaseConstant.createIntConstant(0); - expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); - break; - case 6: - OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; - ifArgs[0] = OceanBaseConstant.createNullConstant(); - ifArgs[1] = isTrue; - expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); - break; - case 7: - OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; - coalesceArgs[0] = OceanBaseConstant.createNullConstant(); - coalesceArgs[1] = isTrue; - expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); - break; + + private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition) { + // we can treat "is true" as combinations of "is flase" and "not","is not true" and "not",etc. + OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE, false); + + OceanBaseUnaryPostfixOperation isFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, false); + + OceanBaseUnaryPostfixOperation isNotFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, true); + + OceanBaseUnaryPostfixOperation isNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + + OceanBaseUnaryPostfixOperation isNotNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, true); + + OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); + Option a = Randomly.fromOptions(Option.values()); + switch (a) { + case TRUE: + expr = isTrue; + break; + case FALSE_NULL: + // not((is false) or (is null)) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseBinaryLogicalOperation(isFalse, isNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_NOT_TRUE: + // not(not(is true))) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseUnaryPrefixOperation(isTrue, + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_FALSE_NOT_NULL: + // (is not false) and (is not null) + expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); + break; + case IF: + // if(1, xx is true, 0) + OceanBaseExpression[] args = new OceanBaseExpression[3]; + args[0] = OceanBaseConstant.createIntConstant(1); + args[1] = isTrue; + args[2] = OceanBaseConstant.createIntConstant(0); + expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); + break; + case IFNULL: + // ifnull(null, xx is true) + OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; + ifArgs[0] = OceanBaseConstant.createNullConstant(); + ifArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); + break; + case COALESCE: + // coalesce(null, xx is true) + OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; + coalesceArgs[0] = OceanBaseConstant.createNullConstant(); + coalesceArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); + break; + default: + expr = isTrue; + break; } return expr; } - } diff --git a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java index 0b9c04040..48fe1002e 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java @@ -1,9 +1,9 @@ package sqlancer.oceanbase.oracle; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.ArrayList; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -21,14 +21,14 @@ import sqlancer.oceanbase.ast.OceanBaseColumnReference; import sqlancer.oceanbase.ast.OceanBaseConstant; import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; +import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseTableReference; import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation.UnaryPostfixOperator; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; -import sqlancer.oceanbase.ast.OceanBaseOrderByTerm; -import sqlancer.oceanbase.ast.OceanBaseOrderByTerm.OceanBaseOrder; import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; public class OceanBasePivotedQuerySynthesisOracle @@ -40,7 +40,7 @@ public class OceanBasePivotedQuerySynthesisOracle public OceanBasePivotedQuerySynthesisOracle(OceanBaseGlobalState globalState) throws SQLException { super(globalState); OceanBaseErrors.addExpressionErrors(errors); - errors.add("in 'order clause'"); + errors.add("in 'order clause'"); errors.add("value is out of range"); } @@ -54,9 +54,11 @@ public Query getRectifiedQuery() throws SQLException { columns = randomFromTables.getColumns(); pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); - selectStatement.setFromList(tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList())); + selectStatement + .setFromList(tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList())); - fetchColumns = columns.stream().map(c -> new OceanBaseColumnReference(c, null)).map(d -> d.setRef(true)).collect(Collectors.toList()); + fetchColumns = columns.stream().map(c -> new OceanBaseColumnReference(c, null)).map(d -> d.setRef(true)) + .collect(Collectors.toList()); selectStatement.setFetchColumns(fetchColumns); OceanBaseExpression whereClause = generateRectifiedExpression(columns, pivotRow); selectStatement.setWhereClause(whereClause); @@ -135,11 +137,13 @@ protected Query getContainmentCheckQuery(Query query) throws S if (i++ != 0) { sb.append(" AND "); } - if(pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) - sb.append("concat("); + if (pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) { + sb.append("concat("); + } sb.append("result." + c.getTable().getName() + c.getName()); - if(pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) - sb.append(",'')"); + if (pivotRow.getValues().get(c) instanceof OceanBaseConstant.OceanBaseTextConstant) { + sb.append(",'')"); + } if (pivotRow.getValues().get(c).isNull()) { sb.append(" IS NULL"); } else { diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java index 3e05a305b..51f06a3fc 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java @@ -2,26 +2,26 @@ import java.sql.SQLException; import java.util.Arrays; -import java.util.HashSet; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; -import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTables; -import sqlancer.oceanbase.ast.*; -import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; +import sqlancer.oceanbase.ast.OceanBaseColumnReference; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.ast.OceanBaseTableReference; import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; import sqlancer.oceanbase.gen.OceanBaseHintGenerator; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -public abstract class OceanBaseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { +public abstract class OceanBaseTLPBase + extends TernaryLogicPartitioningOracleBase implements TestOracle { OceanBaseSchema s; OceanBaseTables targetTables; @@ -53,7 +53,7 @@ public void check() throws SQLException { List generateFetchColumns() { return Arrays.asList(OceanBaseColumnReference.create(targetTables.getColumns().get(0), null)); } - + @Override protected ExpressionGenerator getGen() { return gen; diff --git a/test/sqlancer/dbms/TestOceanBaseNoREC.java b/test/sqlancer/dbms/TestOceanBaseNoREC.java index ddf6bcda0..9c650265c 100644 --- a/test/sqlancer/dbms/TestOceanBaseNoREC.java +++ b/test/sqlancer/dbms/TestOceanBaseNoREC.java @@ -17,12 +17,12 @@ public void testNoREC() { assumeTrue(oceanBaseIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--database-prefix", - "pqsdb", "--num-queries", - TestConfig.NUM_QUERIES, - "--username","sqlancer@test", "--password", "sqlancer", - //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "--num-threads", "4", "--database-prefix", "norecdb", "--num-queries", TestConfig.NUM_QUERIES, + "--username", "sqlancer@test", "--password", "sqlancer", + // after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then + // create user sqlancer: + // mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by + // 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "NoREC" })); } diff --git a/test/sqlancer/dbms/TestOceanBasePQS.java b/test/sqlancer/dbms/TestOceanBasePQS.java index 4123e76c0..adb2007d5 100644 --- a/test/sqlancer/dbms/TestOceanBasePQS.java +++ b/test/sqlancer/dbms/TestOceanBasePQS.java @@ -17,12 +17,13 @@ public void testPQS() { assumeTrue(oceanBaseIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", - "pqsdb", "--num-queries", - TestConfig.NUM_QUERIES, - "--username","sqlancer@test", "--password", "sqlancer", - //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", + "--database-prefix", "pqsdb", "--num-queries", TestConfig.NUM_QUERIES, "--username", + "sqlancer@test", "--password", "sqlancer", + // after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then + // create user sqlancer: + // mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by + // 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "PQS" })); } diff --git a/test/sqlancer/dbms/TestOceanBaseTLP.java b/test/sqlancer/dbms/TestOceanBaseTLP.java index 6eef3fc91..fe84118e6 100644 --- a/test/sqlancer/dbms/TestOceanBaseTLP.java +++ b/test/sqlancer/dbms/TestOceanBaseTLP.java @@ -17,12 +17,12 @@ public void testTLP() { assumeTrue(oceanBaseIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--database-prefix", - "pqsdb", "--num-queries", - TestConfig.NUM_QUERIES, - "--username","sqlancer@test", "--password", "sqlancer", - //after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then create user sqlancer: - //mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by 'sqlancer';grant all on *.* to sqlancer;" + "--num-threads", "4", "--database-prefix", "tlpdb", "--num-queries", TestConfig.NUM_QUERIES, + "--username", "sqlancer@test", "--password", "sqlancer", + // after deploy oceanbase,if you don't create tenant to test,firstly create tenant test,then + // create user sqlancer: + // mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"create user sqlancer identified by + // 'sqlancer';grant all on *.* to sqlancer;" "oceanbase", "--oracle", "TLP_WHERE" })); } From 473aec4b10e3030c186d0d49736997f74ee41566 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 18 Dec 2021 11:49:55 +0100 Subject: [PATCH 042/856] Update H2 --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac460d4be..2cf0a2d07 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -152,8 +152,8 @@ jobs: git clone https://github.com/h2database/h2database cd h2database/h2 ./build.sh jar - cp bin/h2-2.0.201.jar ../../target/lib/h2-1.4.200.jar - cp bin/h2-2.0.201.jar ~/.m2/repository/com/h2database/h2/1.4.200/h2-1.4.200.jar + cp bin/h2-2.0.203-SNAPSHOT.jar ../../target/lib/h2-1.4.200.jar + cp bin/h2-2.0.203-SNAPSHOT.jar ~/.m2/repository/com/h2database/h2/1.4.200/h2-1.4.200.jar ls -lh ../../target/lib - name: Run Tests run: H2_AVAILABLE=true mvn -Dtest=TestH2 test From f06df58ca2b7ca600634b1f00b0ccb25c3905112 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 18:49:27 +0000 Subject: [PATCH 043/856] Bump ecj from 3.27.0 to 3.28.0 Bumps ecj from 3.27.0 to 3.28.0. --- updated-dependencies: - dependency-name: org.eclipse.jdt:ecj dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9a998533e..5f8c44003 100644 --- a/pom.xml +++ b/pom.xml @@ -87,7 +87,7 @@ org.eclipse.jdt ecj - 3.27.0 + 3.28.0 From f2940d54af4d32b623f3568d171d177efadf5f28 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 19:02:25 +0000 Subject: [PATCH 044/856] Bump formatter-maven-plugin from 2.17.0 to 2.17.1 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.17.0 to 2.17.1. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.17.0...formatter-maven-plugin-2.17.1) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5f8c44003..4b68b6ab2 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.17.0 + 2.17.1 eclipseformat From 098209f6dc10de7d5b9d6c05c34d0c7d3aee6df3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 19:28:34 +0000 Subject: [PATCH 045/856] Bump junit-jupiter-engine from 5.8.1 to 5.8.2 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.8.1 to 5.8.2. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.8.1...r5.8.2) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4b68b6ab2..10700e923 100644 --- a/pom.xml +++ b/pom.xml @@ -248,7 +248,7 @@ org.junit.jupiter junit-jupiter-engine - 5.8.1 + 5.8.2 test From f12363af9d2fdf57c6a2bd9bc9e0dfd4cbb0ebcb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Dec 2021 19:59:04 +0000 Subject: [PATCH 046/856] Bump h2 from 1.4.200 to 2.0.202 Bumps [h2](https://github.com/h2database/h2database) from 1.4.200 to 2.0.202. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-1.4.200...version-2.0.202) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 10700e923..31a3552c4 100644 --- a/pom.xml +++ b/pom.xml @@ -264,7 +264,7 @@ com.h2database h2 - 1.4.200 + 2.0.202 org.mongodb From b469d181105cd2d2ce7c3c7cbd831c41b7259f1a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 18 Dec 2021 21:33:14 +0100 Subject: [PATCH 047/856] No longer update H2 to the latest version Previously, H2 had not been updated in a long time. Currently, updating to the latest version is no longer necessary. --- .github/workflows/main.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cf0a2d07..e11a8d053 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -147,14 +147,6 @@ jobs: java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up H2 - run: | - git clone https://github.com/h2database/h2database - cd h2database/h2 - ./build.sh jar - cp bin/h2-2.0.203-SNAPSHOT.jar ../../target/lib/h2-1.4.200.jar - cp bin/h2-2.0.203-SNAPSHOT.jar ~/.m2/repository/com/h2database/h2/1.4.200/h2-1.4.200.jar - ls -lh ../../target/lib - name: Run Tests run: H2_AVAILABLE=true mvn -Dtest=TestH2 test From 6a6ab923a2e699c0e3a61cdd169c6e45e0342faa Mon Sep 17 00:00:00 2001 From: bashaojing Date: Wed, 22 Dec 2021 16:40:21 +0800 Subject: [PATCH 048/856] fix some build warnings --- .../oceanbase/OceanBaseUserCheckException.java | 2 ++ .../oceanbase/ast/OceanBaseComputableFunction.java | 12 ++++++------ src/sqlancer/oceanbase/ast/OceanBaseConstant.java | 8 +++----- .../gen/datadef/OceanBaseIndexGenerator.java | 4 ---- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/oceanbase/OceanBaseUserCheckException.java b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java index 3aedc6f48..38136fc8a 100644 --- a/src/sqlancer/oceanbase/OceanBaseUserCheckException.java +++ b/src/sqlancer/oceanbase/OceanBaseUserCheckException.java @@ -2,6 +2,8 @@ public class OceanBaseUserCheckException extends RuntimeException { + private static final long serialVersionUID = 1L; + public OceanBaseUserCheckException(String s) { super(s); } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java index e9dd03ebe..0ac995d32 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseComputableFunction.java @@ -30,7 +30,7 @@ public enum OceanBaseFunction { BIT_COUNT(1, "BIT_COUNT") { @Override - public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args) { OceanBaseConstant arg = evaluatedArgs[0]; if (arg.isNull()) { return OceanBaseConstant.createNullConstant(); @@ -44,7 +44,7 @@ public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpre COALESCE(2, "COALESCE") { @Override - public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression... origArgs) { OceanBaseConstant result = OceanBaseConstant.createNullConstant(); for (OceanBaseConstant arg : args) { if (!arg.isNull()) { @@ -64,7 +64,7 @@ public boolean isVariadic() { IF(3, "IF") { @Override - public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression... origArgs) { OceanBaseConstant cond = args[0]; OceanBaseConstant left = args[1]; OceanBaseConstant right = args[2]; @@ -81,7 +81,7 @@ public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] o IFNULL(2, "IFNULL") { @Override - public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] origArgs) { + public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression... origArgs) { OceanBaseConstant result; if (args[0].isNull()) { result = args[1]; @@ -95,14 +95,14 @@ public OceanBaseConstant apply(OceanBaseConstant[] args, OceanBaseExpression[] o LEAST(2, "LEAST", true) { @Override - public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args) { return aggregate(evaluatedArgs, args, (min, cur) -> cur.isLessThan(min).asBooleanNotNull() ? cur : min); } }, GREATEST(2, "GREATEST", true) { @Override - public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression[] args) { + public OceanBaseConstant apply(OceanBaseConstant[] evaluatedArgs, OceanBaseExpression... args) { return aggregate(evaluatedArgs, args, (max, cur) -> cur.isLessThan(max).asBooleanNotNull() ? max : cur); } }; diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java index 174defdb8..779d725b1 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -93,7 +93,7 @@ public double getDouble() { @Override public long getInt() { - return new Double(val).longValue(); + return (long) val; } @Override @@ -104,13 +104,13 @@ public boolean asBooleanNotNull() { @Override public OceanBaseConstant castAs(CastType type) { if (type == CastType.SIGNED) { - long value = new Double(val).longValue(); + long value = (long) val; if (val - value >= 0.5) { value = value + 1; } return new OceanBaseIntConstant(value, true); } else if (type == CastType.UNSIGNED) { - long value = new Double(val).longValue(); + long value = (long) val; if (val - value >= 0.5) { value = value + 1; } @@ -284,7 +284,6 @@ public OceanBaseConstant castAs(CastType type) { long val = Long.parseLong(substring); return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); } catch (NumberFormatException e) { - throw new IgnoreMeException(); } } return OceanBaseConstant.createIntConstant(0, type == CastType.SIGNED ? true : false); @@ -308,7 +307,6 @@ public OceanBaseConstant castAsDouble() { double val = Double.parseDouble(substring); return OceanBaseConstant.createDoubleConstant(val); } catch (NumberFormatException e) { - throw new IgnoreMeException(); } } return OceanBaseConstant.createIntConstant(0); diff --git a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java index b7812a7ff..726d9223d 100644 --- a/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java +++ b/src/sqlancer/oceanbase/gen/datadef/OceanBaseIndexGenerator.java @@ -82,10 +82,6 @@ public SQLQueryAdapter create() { return new SQLQueryAdapter(string, errors, true); } - private enum PartitionOptions { - HASH - } - private void appendPartitionOptions() { if (Randomly.getBoolean()) { return; From 85297ba89c7f58ebfbf2613027cac558695a0fbe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Dec 2021 16:01:15 +0000 Subject: [PATCH 049/856] Bump spotbugs-maven-plugin from 4.5.0.0 to 4.5.2.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.5.0.0 to 4.5.2.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.5.0.0...spotbugs-maven-plugin-4.5.2.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 31a3552c4..e723a791e 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.5.0.0 + 4.5.2.0 spotbugs From cedfd697c7933771f8fef63976877c7bdcb97519 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Dec 2021 09:12:10 +0000 Subject: [PATCH 050/856] Bump h2 from 2.0.202 to 2.0.204 Bumps [h2](https://github.com/h2database/h2database) from 2.0.202 to 2.0.204. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.0.202...version-2.0.204) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e723a791e..95871af6a 100644 --- a/pom.xml +++ b/pom.xml @@ -264,7 +264,7 @@ com.h2database h2 - 2.0.202 + 2.0.204 org.mongodb From c842f2ffd9010423f3698f6f8a377a956fd9dba3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Dec 2021 12:07:03 +0100 Subject: [PATCH 051/856] Remove the obsolete H2 UNDO_LOG option --- src/sqlancer/h2/H2SetGenerator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sqlancer/h2/H2SetGenerator.java b/src/sqlancer/h2/H2SetGenerator.java index 88833e1e9..12e3b39eb 100644 --- a/src/sqlancer/h2/H2SetGenerator.java +++ b/src/sqlancer/h2/H2SetGenerator.java @@ -33,8 +33,7 @@ private enum Option { OPTIMIZE_REUSE_RESULTS((r) -> Randomly.fromOptions(0, 1)), // QUERY_STATISTICS((r) -> Randomly.fromOptions("TRUE", "FALSE")), // QUERY_STATISTICS_MAX_ENTRIES((r) -> Randomly.getNotCachedInteger(0, 100000)), // - REFERENTIAL_INTEGRITY((r) -> Randomly.fromOptions("TRUE", "FALSE")), // - UNDO_LOG((r) -> Randomly.fromOptions("TRUE", "FALSE")); + REFERENTIAL_INTEGRITY((r) -> Randomly.fromOptions("TRUE", "FALSE")); // private Function prod; From 8ef43e3456caec7b5469c0062618684e88ce54a0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 31 Dec 2021 09:11:31 +0000 Subject: [PATCH 052/856] Bump clickhouse-jdbc from 0.3.1-patch to 0.3.2 Bumps [clickhouse-jdbc](https://github.com/ClickHouse/clickhouse-jdbc) from 0.3.1-patch to 0.3.2. - [Release notes](https://github.com/ClickHouse/clickhouse-jdbc/releases) - [Changelog](https://github.com/ClickHouse/clickhouse-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/ClickHouse/clickhouse-jdbc/compare/v0.3.1-patch...v0.3.2) --- updated-dependencies: - dependency-name: ru.yandex.clickhouse:clickhouse-jdbc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 95871af6a..97a948316 100644 --- a/pom.xml +++ b/pom.xml @@ -259,7 +259,7 @@ ru.yandex.clickhouse clickhouse-jdbc - 0.3.1-patch + 0.3.2 com.h2database From 1f0aabc7e57c2ce6ce0dd5d5220637008fa5b047 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Jan 2022 09:10:16 +0000 Subject: [PATCH 053/856] Bump h2 from 2.0.204 to 2.0.206 Bumps [h2](https://github.com/h2database/h2database) from 2.0.204 to 2.0.206. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.0.204...version-2.0.206) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97a948316..a40f09afc 100644 --- a/pom.xml +++ b/pom.xml @@ -264,7 +264,7 @@ com.h2database h2 - 2.0.204 + 2.0.206 org.mongodb From b198033cdea7f22de04fd733ac2d715fcf9fb483 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 09:15:41 +0000 Subject: [PATCH 054/856] Bump maven-jar-plugin from 3.2.0 to 3.2.1 Bumps [maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 3.2.0 to 3.2.1. - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.0...maven-jar-plugin-3.2.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a40f09afc..b1e66e1c1 100644 --- a/pom.xml +++ b/pom.xml @@ -127,7 +127,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.0 + 3.2.1 From ddf45799dc84e45781d6ccc8b4ecd81567380dfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 12:37:08 +0000 Subject: [PATCH 055/856] Bump spotbugs-maven-plugin from 4.5.2.0 to 4.5.3.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.5.2.0 to 4.5.3.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.5.2.0...spotbugs-maven-plugin-4.5.3.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b1e66e1c1..f1c20c736 100644 --- a/pom.xml +++ b/pom.xml @@ -196,7 +196,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.5.2.0 + 4.5.3.0 spotbugs From 55f718c6a1cd593b4e4440ab90dc0c5845e24245 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jan 2022 09:13:39 +0000 Subject: [PATCH 056/856] Bump jcommander from 1.81 to 1.82 Bumps [jcommander](https://github.com/cbeust/jcommander) from 1.81 to 1.82. - [Release notes](https://github.com/cbeust/jcommander/releases) - [Changelog](https://github.com/cbeust/jcommander/blob/master/CHANGELOG.md) - [Commits](https://github.com/cbeust/jcommander/compare/1.81...1.82) --- updated-dependencies: - dependency-name: com.beust:jcommander dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f1c20c736..c48c7ae1b 100644 --- a/pom.xml +++ b/pom.xml @@ -218,7 +218,7 @@ com.beust jcommander - 1.81 + 1.82 org.postgresql From 9b48d6da9a829faea0aecc2e31ee415ccfe1116c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Oct 2021 09:10:15 +0000 Subject: [PATCH 057/856] Bump plexus-compiler-eclipse from 2.8.8 to 2.9.0 Bumps plexus-compiler-eclipse from 2.8.8 to 2.9.0. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c48c7ae1b..146ab3ff2 100644 --- a/pom.xml +++ b/pom.xml @@ -82,7 +82,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.8.8 + 2.9.0 org.eclipse.jdt From 21e4ac3e1640b03fd16b115faa51297da2ee2efc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 11 Jan 2022 23:51:21 +0100 Subject: [PATCH 058/856] Add a plexus-compiler-related dependency to avoid an error See https://github.com/codehaus-plexus/plexus-compiler/issues/162. --- pom.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 146ab3ff2..eb2a81eeb 100644 --- a/pom.xml +++ b/pom.xml @@ -69,8 +69,8 @@ maven-compiler-plugin 3.8.1 - 8 - 8 + 1.8 + 1.8 ${project.basedir}/.settings/org.eclipse.jdt.core.prefs @@ -89,6 +89,11 @@ ecj 3.28.0 + + org.codehaus.plexus + plexus-compiler-api + 2.9.0 + From ec5af7f17902e56374f4ddc9304252bd0635ea65 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Jan 2022 21:20:50 +0100 Subject: [PATCH 059/856] Update the version to 2.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eb2a81eeb..b57ed0106 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.sqlancer sqlancer - 1.1.0 + 2.0.0 SQLancer http://www.sqlancer.com/ SQLancer finds logic bugs in Database Management Systems through automatic testing From 8774dc4b8d659271ae218830983c6c34b2593225 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Jan 2022 22:00:15 +0100 Subject: [PATCH 060/856] Enable manually publishing releases --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7777bdb6c..0ce243919 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,6 +2,7 @@ name: Publish package to the Maven Central Repository and Docker Hub on: release: types: [created] + workflow_dispatch: jobs: publish: runs-on: ubuntu-latest From 9571c574b8cd70bd5df745441adc3ccbc37e624f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Jan 2022 22:22:46 +0100 Subject: [PATCH 061/856] Update to Java 11 This is expected to fix an error when executing the release GitHub Action: Failed to execute goal org.apache.maven.plugins:maven-compiler-plugin:3.8.1:compile (default-compile) on project sqlancer: Execution default-compile of goal org.apache.maven.plugins:maven-compiler-plugin:3.8.1:compile failed: An API incompatibility was encountered while executing org.apache.maven.plugins:maven-compiler-plugin:3.8.1:compile: java.lang.UnsupportedClassVersionError: org/eclipse/jdt/core/compiler/CompilationProgress has been compiled by a more recent version of the Java Runtime (class file version 55.0), this version of the Java Runtime only recognizes class file versions up to 52.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index b57ed0106..4c84a6b07 100644 --- a/pom.xml +++ b/pom.xml @@ -69,8 +69,8 @@ maven-compiler-plugin 3.8.1 - 1.8 - 1.8 + 11 + 11 ${project.basedir}/.settings/org.eclipse.jdt.core.prefs From 0e549f6085c0df8bbb2a8368287d8cd16508f6b5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 12 Jan 2022 22:49:05 +0100 Subject: [PATCH 062/856] Also update the release GitHub Actions Java version to 11 --- .github/workflows/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0ce243919..342a358f3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: - name: Set up Maven Central Repository uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD @@ -28,10 +28,10 @@ jobs: steps: - name: Check out the repo uses: actions/checkout@v2 - - name: Set up JDK 1.8 + - name: Set up JDK 11 uses: actions/setup-java@v1 with: - java-version: 1.8 + java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Push to Docker Hub From 14f02a11bfc30d11ac6885bbf7b34b53eb712b31 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 09:10:14 +0000 Subject: [PATCH 063/856] Bump maven-compiler-plugin from 3.8.1 to 3.9.0 Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.8.1 to 3.9.0. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.8.1...maven-compiler-plugin-3.9.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4c84a6b07..fe50b6321 100644 --- a/pom.xml +++ b/pom.xml @@ -67,7 +67,7 @@ maven-compiler-plugin - 3.8.1 + 3.9.0 11 11 From d343ac7f5e3bb4e31e1ad51108d55f442b94909b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 09:48:09 +0000 Subject: [PATCH 064/856] Bump maven-jar-plugin from 3.2.1 to 3.2.2 Bumps [maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 3.2.1 to 3.2.2. - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.1...maven-jar-plugin-3.2.2) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fe50b6321..d7e63786c 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.1 + 3.2.2 From 02544435e90a9b60296f2dc368fdbb8d1bd7f992 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 13 Jan 2022 00:07:36 +0100 Subject: [PATCH 065/856] Directly include the JCommander classes in the SQLancer JAR for better usability See https://github.com/sqlancer/sqlancer/issues/299. Do not include the JDBC drivers so that they can be easily replaced. --- pom.xml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/pom.xml b/pom.xml index d7e63786c..4c115d93d 100644 --- a/pom.xml +++ b/pom.xml @@ -41,6 +41,26 @@ src test + + org.apache.maven.plugins + maven-shade-plugin + 1.6 + + + package + + shade + + + + + com.beust:jcommander + + + + + + org.apache.maven.plugins maven-surefire-plugin @@ -125,6 +145,7 @@ false false true + jcommander From c46a1e81e108215a343fe38bcb1cc2411490ff2d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 14 Jan 2022 00:14:07 +0100 Subject: [PATCH 066/856] Force re-creating the JAR on every build This prevents warnings such as the following when the JAR already exists: [WARNING] We have a duplicate com/beust/ah/A.class in /home/manuel/.m2/repository/com/beust/jcommander/1.82/jcommander-1.82.jar --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 4c115d93d..7100f0dc7 100644 --- a/pom.xml +++ b/pom.xml @@ -155,6 +155,7 @@ maven-jar-plugin 3.2.2 + true true From bb013028ebd0a0200c4e1d130397736f52d0954b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 14 Jan 2022 00:28:46 +0100 Subject: [PATCH 067/856] Delete the backed-up original-sqlancer-*.jar file created by the shade plugin --- pom.xml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pom.xml b/pom.xml index 7100f0dc7..097828323 100644 --- a/pom.xml +++ b/pom.xml @@ -61,6 +61,26 @@ + + org.apache.maven.plugins + maven-antrun-plugin + 1.8 + + + package + + run + + + + + + + + + + + org.apache.maven.plugins maven-surefire-plugin From 9de71b1d21c5089567cee3b116f588ccf78f521b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 09:14:35 +0000 Subject: [PATCH 068/856] Bump maven-antrun-plugin from 1.8 to 3.0.0 Bumps [maven-antrun-plugin](https://github.com/apache/maven-antrun-plugin) from 1.8 to 3.0.0. - [Release notes](https://github.com/apache/maven-antrun-plugin/releases) - [Commits](https://github.com/apache/maven-antrun-plugin/compare/maven-antrun-plugin-1.8...maven-antrun-plugin-3.0.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-antrun-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 097828323..f4b4150b2 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ org.apache.maven.plugins maven-antrun-plugin - 1.8 + 3.0.0 package From c51fb20fa02b9343d000c2434c88557dde76d90a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 09:28:18 +0000 Subject: [PATCH 069/856] Bump maven-shade-plugin from 1.6 to 3.2.4 Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 1.6 to 3.2.4. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-1.6...maven-shade-plugin-3.2.4) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-shade-plugin dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f4b4150b2..0a1930e9f 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ org.apache.maven.plugins maven-shade-plugin - 1.6 + 3.2.4 package From 7d69a1d42be348150c407638f296f20bc0817572 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 09:44:03 +0000 Subject: [PATCH 070/856] Bump slf4j-simple from 1.7.32 to 1.7.33 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.32 to 1.7.33. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.32...v_1.7.33) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0a1930e9f..e35b7b3d2 100644 --- a/pom.xml +++ b/pom.xml @@ -301,7 +301,7 @@ org.slf4j slf4j-simple - 1.7.32 + 1.7.33 ru.yandex.clickhouse From 79681e323ba18dc8943d74eb13f6bf8734087154 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 17 Jan 2022 23:10:41 +0100 Subject: [PATCH 071/856] Add missing right parentheses in option descriptions --- src/sqlancer/clickhouse/ClickHouseOptions.java | 2 +- src/sqlancer/cockroachdb/CockroachDBOptions.java | 2 +- src/sqlancer/mariadb/MariaDBOptions.java | 2 +- src/sqlancer/mysql/MySQLOptions.java | 2 +- src/sqlancer/oceanbase/OceanBaseOptions.java | 2 +- src/sqlancer/postgres/PostgresOptions.java | 2 +- src/sqlancer/tidb/TiDBOptions.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index c25905648..c58697c98 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -19,7 +19,7 @@ import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "ClickHouse (default port: " + ClickHouseOptions.DEFAULT_PORT - + ", default host: " + ClickHouseOptions.DEFAULT_HOST) + + ", default host: " + ClickHouseOptions.DEFAULT_HOST + ")") public class ClickHouseOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 8123; diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 7673c4c0a..f8ab8e07f 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -23,7 +23,7 @@ import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "CockroachDB (default port: " + CockroachDBOptions.DEFAULT_PORT - + " default host: " + CockroachDBOptions.DEFAULT_HOST) + + " default host: " + CockroachDBOptions.DEFAULT_HOST + ")") public class CockroachDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 26257; diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 63e142aec..02e6eab64 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -15,7 +15,7 @@ import sqlancer.mariadb.oracle.MariaDBNoRECOracle; @Parameters(separators = "=", commandDescription = "MariaDB (default port: " + MariaDBOptions.DEFAULT_PORT - + ", default host: " + MariaDBOptions.DEFAULT_HOST) + + ", default host: " + MariaDBOptions.DEFAULT_HOST + ")") public class MariaDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 5540f00a8..2a3f16968 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -15,7 +15,7 @@ import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @Parameters(separators = "=", commandDescription = "MySQL (default port: " + MySQLOptions.DEFAULT_PORT - + ", default host: " + MySQLOptions.DEFAULT_HOST) + + ", default host: " + MySQLOptions.DEFAULT_HOST + ")") public class MySQLOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index eb7200437..b19de3983 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -16,7 +16,7 @@ import sqlancer.oceanbase.oracle.OceanBaseTLPWhereOracle; @Parameters(separators = "=", commandDescription = "OceanBase (default port: " + OceanBaseOptions.DEFAULT_PORT - + ", default host: " + OceanBaseOptions.DEFAULT_HOST) + + ", default host: " + OceanBaseOptions.DEFAULT_HOST + ")") public class OceanBaseOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 2881; diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 6c0a97847..ed2856451 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -20,7 +20,7 @@ import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; @Parameters(separators = "=", commandDescription = "PostgreSQL (default port: " + PostgresOptions.DEFAULT_PORT - + ", default host: " + PostgresOptions.DEFAULT_HOST) + + ", default host: " + PostgresOptions.DEFAULT_HOST + ")") public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index e5da34fe0..f9137f6f7 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -18,7 +18,7 @@ import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @Parameters(separators = "=", commandDescription = "TiDB (default port: " + TiDBOptions.DEFAULT_PORT - + ", default host: " + TiDBOptions.DEFAULT_HOST) + + ", default host: " + TiDBOptions.DEFAULT_HOST + ")") public class TiDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 4000; From 298bab1d418bb00407bfafe53fa48e917caa7d48 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 17 Jan 2022 23:20:37 +0100 Subject: [PATCH 072/856] Add a description to database implementations without any --- src/sqlancer/arangodb/ArangoDBOptions.java | 2 ++ src/sqlancer/duckdb/DuckDBOptions.java | 2 +- src/sqlancer/h2/H2Options.java | 3 +++ src/sqlancer/mongodb/MongoDBOptions.java | 2 ++ 4 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index 6e7a607f0..b821fe8c3 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -7,6 +7,7 @@ import java.util.List; import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; @@ -15,6 +16,7 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; +@Parameters(commandDescription = "ArangoDB (experimental)") public class ArangoDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index c163b5d0b..4e1c49cd1 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -21,7 +21,7 @@ import sqlancer.duckdb.test.DuckDBQueryPartitioningHavingTester; import sqlancer.duckdb.test.DuckDBQueryPartitioningWhereTester; -@Parameters +@Parameters(commandDescription = "DuckDB") public class DuckDBOptions implements DBMSSpecificOptions { @Parameter(names = "--test-collate", arity = 1) diff --git a/src/sqlancer/h2/H2Options.java b/src/sqlancer/h2/H2Options.java index 3ae576e1c..45b47f203 100644 --- a/src/sqlancer/h2/H2Options.java +++ b/src/sqlancer/h2/H2Options.java @@ -4,12 +4,15 @@ import java.util.Arrays; import java.util.List; +import com.beust.jcommander.Parameters; + import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; import sqlancer.h2.H2Options.H2OracleFactory; import sqlancer.h2.H2Provider.H2GlobalState; +@Parameters(commandDescription = "H2") public class H2Options implements DBMSSpecificOptions { public enum H2OracleFactory implements OracleFactory { diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 5c618f5f7..9e02de67b 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -8,6 +8,7 @@ import java.util.List; import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; @@ -17,6 +18,7 @@ import sqlancer.mongodb.test.MongoDBDocumentRemovalTester; import sqlancer.mongodb.test.MongoDBQueryPartitioningWhereTester; +@Parameters(commandDescription = "MongoDB (experimental)") public class MongoDBOptions implements DBMSSpecificOptions { @Parameter(names = "--test-validation", description = "Enable/Disable validation of schema with Schema Validation", arity = 1) From a4756aedac3377925c72842827ba5ca28d51a68e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 17 Jan 2022 23:28:52 +0100 Subject: [PATCH 073/856] Check the usage naming conventions --- src/sqlancer/Main.java | 3 -- test/sqlancer/TestUsageNamingConvention.java | 34 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 test/sqlancer/TestUsageNamingConvention.java diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a3ff789e4..c499ef9ec 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -404,9 +404,6 @@ public static int executeMain(String... args) throws AssertionError { Builder commandBuilder = JCommander.newBuilder().addObject(options); for (DatabaseProvider provider : providers) { String name = provider.getDBMSName(); - if (!name.toLowerCase().equals(name)) { - throw new AssertionError(name + " should be in lowercase!"); - } DBMSExecutorFactory executorFactory = new DBMSExecutorFactory<>(provider, options); commandBuilder = commandBuilder.addCommand(name, executorFactory.getCommand()); nameToProvider.put(name, executorFactory); diff --git a/test/sqlancer/TestUsageNamingConvention.java b/test/sqlancer/TestUsageNamingConvention.java new file mode 100644 index 000000000..94f5daba0 --- /dev/null +++ b/test/sqlancer/TestUsageNamingConvention.java @@ -0,0 +1,34 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import com.beust.jcommander.JCommander; +import com.beust.jcommander.JCommander.Builder; +import com.beust.jcommander.Parameters; + +public class TestUsageNamingConvention { + + @Test + void testNonEmptyDescription() { + List> providers = Main.getDBMSProviders(); + MainOptions options = new MainOptions(); + Builder commandBuilder = JCommander.newBuilder().addObject(options); + for (DatabaseProvider provider : providers) { + String name = provider.getDBMSName(); + if (!name.toLowerCase().equals(name)) { + throw new AssertionError(name + " should be in lowercase!"); + } + commandBuilder.addCommand(provider.getDBMSName(), provider.getOptionClass()); + Parameters param = provider.getOptionClass().getAnnotation(Parameters.class); + assertNotEquals(null, param, provider.getOptionClass().toString()); + String databaseDescription = param.commandDescription(); + assertNotEquals(null, databaseDescription, "description cannot be empty " + provider.getOptionClass()); + assertNotEquals("", databaseDescription, "description cannot be empty " + provider.getOptionClass()); + } + } + +} From d00361ee457dc67c003471bbef4beccd06b7ceea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 09:12:23 +0000 Subject: [PATCH 074/856] Bump mysql-connector-java from 8.0.27 to 8.0.28 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.27 to 8.0.28. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/commits) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e35b7b3d2..9d17c95c0 100644 --- a/pom.xml +++ b/pom.xml @@ -280,7 +280,7 @@ mysql mysql-connector-java - 8.0.27 + 8.0.28 org.mariadb.jdbc From aee9b692c2862c9c6d7c779b329352777b2d3b81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 12:19:50 +0000 Subject: [PATCH 075/856] Bump h2 from 2.0.206 to 2.1.210 Bumps [h2](https://github.com/h2database/h2database) from 2.0.206 to 2.1.210. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.0.206...version-2.1.210) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9d17c95c0..dc709bda2 100644 --- a/pom.xml +++ b/pom.xml @@ -311,7 +311,7 @@ com.h2database h2 - 2.0.206 + 2.1.210 org.mongodb From 6924ce5d78b732df2f8b2e7cafdca97a8f358d1f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 18 Jan 2022 22:16:16 +0100 Subject: [PATCH 076/856] Remove the Travis configuration and other obsolete files --- .github/workflows~HEAD | 21 ------ .github/workflows~HEAD_0 | 21 ------ .github/workflows~HEAD_1 | 21 ------ .travis.yml | 146 --------------------------------------- codecov.yml | 16 ----- 5 files changed, 225 deletions(-) delete mode 100644 .github/workflows~HEAD delete mode 100644 .github/workflows~HEAD_0 delete mode 100644 .github/workflows~HEAD_1 delete mode 100644 .travis.yml delete mode 100644 codecov.yml diff --git a/.github/workflows~HEAD b/.github/workflows~HEAD deleted file mode 100644 index 0e057d405..000000000 --- a/.github/workflows~HEAD +++ /dev/null @@ -1,21 +0,0 @@ -name: CI tests - -on: [push, pull_request] - -jobs: - misc: - name: Misc tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build - run: mvn package -DskipTests=true - - name: Verify - run: mvn verify -DskipTests=true - - name: Misc Tests - run: mvn '-Dtest=!sqlancer.dbms.**' test diff --git a/.github/workflows~HEAD_0 b/.github/workflows~HEAD_0 deleted file mode 100644 index 0e057d405..000000000 --- a/.github/workflows~HEAD_0 +++ /dev/null @@ -1,21 +0,0 @@ -name: CI tests - -on: [push, pull_request] - -jobs: - misc: - name: Misc tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build - run: mvn package -DskipTests=true - - name: Verify - run: mvn verify -DskipTests=true - - name: Misc Tests - run: mvn '-Dtest=!sqlancer.dbms.**' test diff --git a/.github/workflows~HEAD_1 b/.github/workflows~HEAD_1 deleted file mode 100644 index 0e057d405..000000000 --- a/.github/workflows~HEAD_1 +++ /dev/null @@ -1,21 +0,0 @@ -name: CI tests - -on: [push, pull_request] - -jobs: - misc: - name: Misc tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Build - run: mvn package -DskipTests=true - - name: Verify - run: mvn verify -DskipTests=true - - name: Misc Tests - run: mvn '-Dtest=!sqlancer.dbms.**' test diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 318ea6fa7..000000000 --- a/.travis.yml +++ /dev/null @@ -1,146 +0,0 @@ -dist: bionic -language: java - -script: -- cd src && python check_names.py && cd .. - -cache: - directories: - - target/lib -after_success: - - bash <(curl -s https://codecov.io/bash) -after_failure: - - cat target/pmd.xml -branches: - only: - - master - -matrix: - include: - - name: SQLite3 - jdk : openjdk8 - script: - - mvn -Dtest=TestSQLitePQS test - - mvn -Dtest=TestSQLite3 test - - name: PostgreSQL - jdk : openjdk8 - before_install: - - sudo apt-get update - - sudo apt-get --yes remove postgresql\* - - sudo apt-get install -y postgresql-12 postgresql-client-12 - - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/12/main/postgresql.conf - - sudo cp /etc/postgresql/{10,12}/main/pg_hba.conf - - sudo service postgresql restart 12 - addons: - postgresql: "12.3" - env: - global: - - PGPORT=5432 - services: - - postgresql - before_script: - - sudo apt-get install locales - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -U postgres - - createdb test -U postgres - script: - - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test - - name: MariaDB - jdk : openjdk8 - before_install: - - sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 - - sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main' - - sudo apt update - - sudo apt install mariadb-server - - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - - MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test - - name : MySQL - jdk : openjdk8 - script: - - sudo apt-get update && sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 - - sudo apt-get remove mysql-* && wget https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar && yes | sudo dpkg -i *.deb - - sudo mysql -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS - - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP - - name: CockroachDB - jdk : openjdk8 - before_install: - - wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.2.linux-amd64.tgz | tar xvz - - cd cockroach-v20.1.2.linux-amd64/ && ./cockroach start-single-node --insecure & - - sleep 15 - - cd cockroach-v20.1.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - script: - - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - - name: TiDB - jdk : openjdk8 - services: - - docker - before_install: - - docker pull pingcap/tidb:latest - - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest - - sleep 15 - - sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - script: - - TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test - - name: DuckDB - jdk : openjdk8 - script: - - mvn -Dtest=TestDuckDB test - - name: DuckDB (Java 8) - jdk : openjdk8 - script: - - mvn -Dtest=TestDuckDB test - - name: DuckDB (java 13) - jdk : openjdk8 - script: - - mvn -Dtest=TestDuckDB test - - name: Misc - jdk : openjdk8 - script: - - mvn '-Dtest=!sqlancer.dbms.**' test - - name: Citus (PostgreSQL extension) - jdk : openjdk8 - before_install: - - sudo apt-get update - - sudo apt-get install -y postgresql-server-dev-12 postgresql-12 locales - - git clone https://github.com/citusdata/citus.git - - cd citus - - ./configure - - make clean -sj9 - - sudo make install -sj9 - - cd .. - before_script: - - sudo chown -R $USER:$USER /var/run/postgresql - - export PATH=/usr/lib/postgresql/12/bin:$PATH - - mkdir -p citus/coordinator citus/worker1 citus/worker2 - - initdb -D citus/coordinator - - initdb -D citus/worker1 - - initdb -D citus/worker2 - - echo "shared_preload_libraries = 'citus'" >> citus/coordinator/postgresql.conf - - echo "shared_preload_libraries = 'citus'" >> citus/worker1/postgresql.conf - - echo "shared_preload_libraries = 'citus'" >> citus/worker2/postgresql.conf - - pg_ctl -D citus/coordinator -o "-p 9700" -l coordinator_logfile start - - pg_ctl -D citus/worker1 -o "-p 9701" -l worker1_logfile start - - pg_ctl -D citus/worker2 -o "-p 9702" -l worker2_logfile start - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9700 -d postgres -U $USER - - createdb test -p 9700 -U $USER - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9701 -d postgres -U $USER - - createdb test -p 9701 -U $USER - - psql -c "CREATE ROLE sqlancer SUPERUSER LOGIN CREATEDB PASSWORD 'sqlancer';" -p 9702 -d postgres -U $USER - - createdb test -p 9702 -U $USER - - psql -c "CREATE EXTENSION citus;" -p 9700 -U $USER -d test - - psql -c "CREATE EXTENSION citus;" -p 9701 -U $USER -d test - - psql -c "CREATE EXTENSION citus;" -p 9702 -U $USER -d test - - psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test - - psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test - script: - - CITUS_AVAILABLE=true mvn -Dtest=TestCitus test - - name: ClickHouse - jdk : openjdk8 - services: - - docker - before_install: - - docker pull yandex/clickhouse-server:latest - - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest - - sleep 5 - script: - - CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test diff --git a/codecov.yml b/codecov.yml deleted file mode 100644 index ed891a44d..000000000 --- a/codecov.yml +++ /dev/null @@ -1,16 +0,0 @@ -codecov: - require_ci_to_pass: no - notify: - after_n_builds: 11 - -coverage: - range: "50...100" - -coverage: - status: - project: - default: - threshold: 2% - patch: - default: - threshold: 2% From f278315dbf040ebb69ca141f73b4584614c66925 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:22:05 +0100 Subject: [PATCH 077/856] Use a ServiceLoader to pick up database testing implementations This change allows SQLancer to pick up database providers in other JARs than the SQLancer JAR, if they are on the classpath. --- pom.xml | 5 +++ src/sqlancer/Main.java | 42 +++++++------------ src/sqlancer/arangodb/ArangoDBProvider.java | 3 ++ src/sqlancer/citus/CitusProvider.java | 4 ++ .../clickhouse/ClickHouseProvider.java | 4 ++ .../cockroachdb/CockroachDBProvider.java | 4 ++ src/sqlancer/cosmos/CosmosProvider.java | 3 ++ src/sqlancer/duckdb/DuckDBProvider.java | 4 ++ src/sqlancer/h2/H2Provider.java | 4 ++ src/sqlancer/mariadb/MariaDBProvider.java | 4 ++ src/sqlancer/mongodb/MongoDBProvider.java | 3 ++ src/sqlancer/mysql/MySQLProvider.java | 4 ++ src/sqlancer/oceanbase/OceanBaseProvider.java | 4 ++ src/sqlancer/postgres/PostgresProvider.java | 4 ++ src/sqlancer/sqlite3/SQLite3Provider.java | 4 ++ src/sqlancer/tidb/TiDBProvider.java | 4 ++ 16 files changed, 72 insertions(+), 28 deletions(-) diff --git a/pom.xml b/pom.xml index dc709bda2..bf7576302 100644 --- a/pom.xml +++ b/pom.xml @@ -262,6 +262,11 @@ + + com.google.auto.service + auto-service + 1.0.1 + com.beust jcommander diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index c499ef9ec..77ecdcabe 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -12,6 +12,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.ServiceLoader; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -21,23 +22,9 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.JCommander.Builder; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.citus.CitusProvider; -import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.cockroachdb.CockroachDBProvider; import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.cosmos.CosmosProvider; -import sqlancer.duckdb.DuckDBProvider; -import sqlancer.h2.H2Provider; -import sqlancer.mariadb.MariaDBProvider; -import sqlancer.mongodb.MongoDBProvider; -import sqlancer.mysql.MySQLProvider; -import sqlancer.oceanbase.OceanBaseProvider; -import sqlancer.postgres.PostgresProvider; -import sqlancer.sqlite3.SQLite3Provider; -import sqlancer.tidb.TiDBProvider; public final class Main { @@ -543,22 +530,21 @@ private boolean run(MainOptions options, ExecutorService execService, return threadsShutdown == 0 ? 0 : options.getErrorExitCode(); } + /** + * To register a new provider, it is necessary to implement the DatabaseProvider interface and add an additional + * configuration file, see https://docs.oracle.com/javase/9/docs/api/java/util/ServiceLoader.html. Currently, we use + * an @AutoService annotation to create the configuration file automatically. This allows SQLancer to pick up + * providers in other JARs on the classpath. + * + * @return The list of service providers on the classpath + */ static List> getDBMSProviders() { List> providers = new ArrayList<>(); - providers.add(new SQLite3Provider()); - providers.add(new CockroachDBProvider()); - providers.add(new MySQLProvider()); - providers.add(new MariaDBProvider()); - providers.add(new TiDBProvider()); - providers.add(new PostgresProvider()); - providers.add(new CitusProvider()); - providers.add(new ClickHouseProvider()); - providers.add(new DuckDBProvider()); - providers.add(new H2Provider()); - providers.add(new MongoDBProvider()); - providers.add(new CosmosProvider()); - providers.add(new ArangoDBProvider()); - providers.add(new OceanBaseProvider()); + @SuppressWarnings("rawtypes") + ServiceLoader loader = ServiceLoader.load(DatabaseProvider.class); + for (DatabaseProvider provider : loader) { + providers.add(provider); + } return providers; } diff --git a/src/sqlancer/arangodb/ArangoDBProvider.java b/src/sqlancer/arangodb/ArangoDBProvider.java index a44c4e422..7bdc2fa01 100644 --- a/src/sqlancer/arangodb/ArangoDBProvider.java +++ b/src/sqlancer/arangodb/ArangoDBProvider.java @@ -5,8 +5,10 @@ import com.arangodb.ArangoDB; import com.arangodb.ArangoDatabase; +import com.google.auto.service.AutoService; import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.ExecutionTimer; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; @@ -19,6 +21,7 @@ import sqlancer.common.log.LoggableFactory; import sqlancer.common.query.Query; +@AutoService(DatabaseProvider.class) public class ArangoDBProvider extends ProviderAdapter { diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 172149a52..aaebd6709 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -10,7 +10,10 @@ import java.util.List; import java.util.stream.Collectors; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -51,6 +54,7 @@ import sqlancer.postgres.gen.PostgresTruncateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; +@AutoService(DatabaseProvider.class) public class CitusProvider extends PostgresProvider { @SuppressWarnings("unchecked") diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index d936238d4..3585aa9df 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -6,7 +6,10 @@ import java.sql.Statement; import java.util.stream.Collectors; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -21,6 +24,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +@AutoService(DatabaseProvider.class) public class ClickHouseProvider extends SQLProviderAdapter { public ClickHouseProvider() { diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 2fcdf1930..abc7e6e20 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -8,6 +8,9 @@ import java.util.List; import java.util.stream.Collectors; +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Main.QueryManager; import sqlancer.MainOptions; @@ -34,6 +37,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +@AutoService(DatabaseProvider.class) public class CockroachDBProvider extends SQLProviderAdapter { public CockroachDBProvider() { diff --git a/src/sqlancer/cosmos/CosmosProvider.java b/src/sqlancer/cosmos/CosmosProvider.java index a8681d3af..424625b4f 100644 --- a/src/sqlancer/cosmos/CosmosProvider.java +++ b/src/sqlancer/cosmos/CosmosProvider.java @@ -1,11 +1,13 @@ package sqlancer.cosmos; +import com.google.auto.service.AutoService; import com.mongodb.ConnectionString; import com.mongodb.MongoClientSettings; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.ProviderAdapter; import sqlancer.Randomly; @@ -17,6 +19,7 @@ import sqlancer.mongodb.MongoDBQueryAdapter; import sqlancer.mongodb.gen.MongoDBTableGenerator; +@AutoService(DatabaseProvider.class) public class CosmosProvider extends ProviderAdapter { diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 73adb3cb0..8453bb6f7 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -3,7 +3,10 @@ import java.sql.DriverManager; import java.sql.SQLException; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -22,6 +25,7 @@ import sqlancer.duckdb.gen.DuckDBUpdateGenerator; import sqlancer.duckdb.gen.DuckDBViewGenerator; +@AutoService(DatabaseProvider.class) public class DuckDBProvider extends SQLProviderAdapter { public DuckDBProvider() { diff --git a/src/sqlancer/h2/H2Provider.java b/src/sqlancer/h2/H2Provider.java index 916849199..7f01e6196 100644 --- a/src/sqlancer/h2/H2Provider.java +++ b/src/sqlancer/h2/H2Provider.java @@ -4,7 +4,10 @@ import java.sql.DriverManager; import java.sql.SQLException; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -15,6 +18,7 @@ import sqlancer.common.query.SQLQueryProvider; import sqlancer.h2.H2Provider.H2GlobalState; +@AutoService(DatabaseProvider.class) public class H2Provider extends SQLProviderAdapter { public H2Provider() { diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index dc0420fca..c84691203 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -7,6 +7,9 @@ import java.util.ArrayList; import java.util.List; +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -24,6 +27,7 @@ import sqlancer.mariadb.gen.MariaDBTruncateGenerator; import sqlancer.mariadb.gen.MariaDBUpdateGenerator; +@AutoService(DatabaseProvider.class) public class MariaDBProvider extends SQLProviderAdapter { public static final int MAX_EXPRESSION_DEPTH = 3; diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java index 3b090dade..09662a863 100644 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ b/src/sqlancer/mongodb/MongoDBProvider.java @@ -3,11 +3,13 @@ import java.util.ArrayList; import java.util.List; +import com.google.auto.service.AutoService; import com.mongodb.client.MongoClient; import com.mongodb.client.MongoClients; import com.mongodb.client.MongoDatabase; import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.ExecutionTimer; import sqlancer.GlobalState; import sqlancer.IgnoreMeException; @@ -21,6 +23,7 @@ import sqlancer.mongodb.gen.MongoDBInsertGenerator; import sqlancer.mongodb.gen.MongoDBTableGenerator; +@AutoService(DatabaseProvider.class) public class MongoDBProvider extends ProviderAdapter { diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 447325951..0f1704bf3 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -5,7 +5,10 @@ import java.sql.SQLException; import java.sql.Statement; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -31,6 +34,7 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLOptimize; import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; +@AutoService(DatabaseProvider.class) public class MySQLProvider extends SQLProviderAdapter { public MySQLProvider() { diff --git a/src/sqlancer/oceanbase/OceanBaseProvider.java b/src/sqlancer/oceanbase/OceanBaseProvider.java index d7308ab63..b7c5794f1 100644 --- a/src/sqlancer/oceanbase/OceanBaseProvider.java +++ b/src/sqlancer/oceanbase/OceanBaseProvider.java @@ -5,7 +5,10 @@ import java.sql.SQLException; import java.sql.Statement; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -24,6 +27,7 @@ import sqlancer.oceanbase.gen.OceanBaseUpdateGenerator; import sqlancer.oceanbase.gen.datadef.OceanBaseIndexGenerator; +@AutoService(DatabaseProvider.class) public class OceanBaseProvider extends SQLProviderAdapter { public OceanBaseProvider() { diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index e0503f2c7..0aeb29328 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -8,7 +8,10 @@ import java.sql.Statement; import java.util.Arrays; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -43,6 +46,7 @@ // EXISTS // IN +@AutoService(DatabaseProvider.class) public class PostgresProvider extends SQLProviderAdapter { /** diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 54f1a6db4..eb88a0e0a 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -7,7 +7,10 @@ import java.util.Arrays; import java.util.List; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -40,6 +43,7 @@ import sqlancer.sqlite3.gen.dml.SQLite3UpdateGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; +@AutoService(DatabaseProvider.class) public class SQLite3Provider extends SQLProviderAdapter { public static boolean allowFloatingPointFp = true; diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index c744de3f2..f0d8f261b 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -5,7 +5,10 @@ import java.sql.SQLException; import java.sql.Statement; +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; import sqlancer.MainOptions; import sqlancer.Randomly; @@ -28,6 +31,7 @@ import sqlancer.tidb.gen.TiDBUpdateGenerator; import sqlancer.tidb.gen.TiDBViewGenerator; +@AutoService(DatabaseProvider.class) public class TiDBProvider extends SQLProviderAdapter { public TiDBProvider() { From a1aa0ff994e98a64ab6cb746561fceda36168c30 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:38:51 +0100 Subject: [PATCH 078/856] Update the Checkstyle version --- pom.xml | 7 +++++++ src/sqlancer/mysql/ast/MySQLConstant.java | 4 ++-- src/sqlancer/oceanbase/ast/OceanBaseConstant.java | 4 ++-- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index bf7576302..97e053869 100644 --- a/pom.xml +++ b/pom.xml @@ -203,6 +203,13 @@ org.apache.maven.plugins maven-checkstyle-plugin 3.1.2 + + + com.puppycrawl.tools + checkstyle + 9.2.1 + + configs/checkstyle.xml diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index 11a807841..2e4922f8e 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -164,12 +164,12 @@ public MySQLConstant castAs(CastType type) { try { String substring = value.substring(0, i); long val = Long.parseLong(substring); - return MySQLConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); + return MySQLConstant.createIntConstant(val, type == CastType.SIGNED); } catch (NumberFormatException e) { // ignore } } - return MySQLConstant.createIntConstant(0, type == CastType.SIGNED ? true : false); + return MySQLConstant.createIntConstant(0, type == CastType.SIGNED); } else { throw new AssertionError(); } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java index 779d725b1..91a481fc6 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -282,11 +282,11 @@ public OceanBaseConstant castAs(CastType type) { try { String substring = value.substring(0, i); long val = Long.parseLong(substring); - return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED ? true : false); + return OceanBaseConstant.createIntConstant(val, type == CastType.SIGNED); } catch (NumberFormatException e) { } } - return OceanBaseConstant.createIntConstant(0, type == CastType.SIGNED ? true : false); + return OceanBaseConstant.createIntConstant(0, type == CastType.SIGNED); } else { throw new AssertionError(); } From f5e2f2ccbcf2fd76599cfe84dc263ef974523a82 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:54:53 +0100 Subject: [PATCH 079/856] Remove the obsolete Codecov badge --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 7008c1d92..dd2e28ee9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ [![Build Status](https://github.com/sqlancer/sqlancer/workflows/ci/badge.svg)](https://github.com/sqlancer/sqlancer/actions) -[![codecov](https://codecov.io/gh/sqlancer/sqlancer/branch/master/graph/badge.svg)](https://codecov.io/gh/sqlancer/sqlancer) [![Twitter](https://img.shields.io/twitter/follow/sqlancer_dbms?style=social)](https://twitter.com/sqlancer_dbms) # SQLancer From 8ab8f994727afc844dfc90ff80a18ae41842a908 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:56:42 +0100 Subject: [PATCH 080/856] Update the requirements --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dd2e28ee9..9ec390fd4 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,9 @@ SQLancer operates in the following two phases: # Getting Started Requirements: -* Java 8 or above +* Java 11 or above * [Maven](https://maven.apache.org/) (`sudo apt install maven` on Ubuntu) -* The DBMS that you want to test (SQLite is an embedded DBMS and is included) +* The DBMS that you want to test (embedded DBMSs such as DuckDB, H2, and SQLite do not require a setup) The following commands clone SQLancer, create a JAR, and start SQLancer to test SQLite using Non-optimizing Reference Engine Construction (NoREC): From 0dda6dfc023d41ed3e763f02c24b7f197864c6ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 19 Jan 2022 22:57:13 +0100 Subject: [PATCH 081/856] Update the note about being a research prototype --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index 9ec390fd4..2209e7589 100644 --- a/README.md +++ b/README.md @@ -33,10 +33,6 @@ If the execution prints progress information every five seconds, then the tool w If you launch SQLancer without parameters, available options and commands are displayed. Note that general options that are supported by all DBMS-testing implementations (e.g., `--num-threads`) need to precede the name of DBMS to be tested (e.g., `sqlite3`). Options that are supported only for specific DBMS (e.g., `--test-rtree` for SQLite3), or options for which each testing implementation provides different values (e.g. `--oracle NoREC`) need to go after the DBMS name. -# Research Prototype - -This project should at this stage still be seen as a research prototype. We believe that the tool is not ready to be used. However, we have received many requests by companies, organizations, and individual developers, which is why we decided to prematurely release the tool. Expect errors, incompatibilities, lack of documentation, and insufficient code quality. That being said, we are working hard to address these issues and enhance SQLancer to become a production-quality piece of software. We welcome any issue reports, extension requests, and code contributions. - # Testing Approaches | Approach | Description | From cf24a9f3dd1584d15a5a3f526d73489425a6ba8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 20 Jan 2022 09:09:54 +0000 Subject: [PATCH 082/856] Bump mariadb-java-client from 2.7.4 to 2.7.5 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.7.4 to 2.7.5. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/2.7.5/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/2.7.4...2.7.5) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 97e053869..f6db87db6 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 2.7.4 + 2.7.5 org.duckdb From 9ffe1d05baed8355650bd3f958b5815f420bcfde Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 20 Jan 2022 12:15:38 +0100 Subject: [PATCH 083/856] Ignore expected Citus error "columnar_parallelscan_estimate not implemented" Closes https://github.com/sqlancer/sqlancer/issues/402. --- src/sqlancer/citus/gen/CitusTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java index c340400f3..86e5d40cf 100644 --- a/src/sqlancer/citus/gen/CitusTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -11,6 +11,8 @@ public CitusTableGenerator(String tableName, PostgresSchema newSchema, boolean g PostgresGlobalState globalState) { super(tableName, newSchema, generateOnlyKnown, globalState); CitusCommon.addCitusErrors(errors); + errors.add("columnar_parallelscan_estimate not implemented"); // see + // https://github.com/sqlancer/sqlancer/issues/402 } public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchema, boolean generateOnlyKnown, From ecdb34a2548238ce2b46551650c11fee76f52071 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Jan 2022 09:10:58 +0000 Subject: [PATCH 084/856] Bump slf4j-simple from 1.7.33 to 1.7.34 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.33 to 1.7.34. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/commits) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f6db87db6..5be2ef0c8 100644 --- a/pom.xml +++ b/pom.xml @@ -313,7 +313,7 @@ org.slf4j slf4j-simple - 1.7.33 + 1.7.34 ru.yandex.clickhouse From 2b08717ed3703b977cd1245cd850126ede02c800 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 09:09:23 +0000 Subject: [PATCH 085/856] Bump mariadb-java-client from 2.7.5 to 3.0.3 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 2.7.5 to 3.0.3. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/2.7.5...3.0.3) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5be2ef0c8..ae96a248b 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 2.7.5 + 3.0.3 org.duckdb From 129d5cc5239a9d2ffd4ed201f12ceeefc6cb8d27 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 10:20:35 +0000 Subject: [PATCH 086/856] Bump slf4j-simple from 1.7.34 to 1.7.35 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.34 to 1.7.35. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.34...v_1.7.35) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ae96a248b..4e9888c2a 100644 --- a/pom.xml +++ b/pom.xml @@ -313,7 +313,7 @@ org.slf4j slf4j-simple - 1.7.34 + 1.7.35 ru.yandex.clickhouse From 140f990af3608b9fca59ea72964c63b84e890c66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jan 2022 09:12:26 +0000 Subject: [PATCH 087/856] Bump checkstyle from 9.2.1 to 9.3 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 9.2.1 to 9.3. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-9.2.1...checkstyle-9.3) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4e9888c2a..67227fd21 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 9.2.1 + 9.3 From 0474249582677b7492d6f5f7fbeceed0c61fab0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 09:13:09 +0000 Subject: [PATCH 088/856] Bump plexus-compiler-eclipse from 2.9.0 to 2.10.0 Bumps plexus-compiler-eclipse from 2.9.0 to 2.10.0. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 67227fd21..e2a1b332f 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.9.0 + 2.10.0 org.eclipse.jdt From 73a8dc0d8bb37d0acb40817b4d9ef59ad4cff052 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 10:45:26 +0000 Subject: [PATCH 089/856] Bump plexus-compiler-api from 2.9.0 to 2.10.0 Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.9.0 to 2.10.0. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.9.0...plexus-compiler-2.10.0) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e2a1b332f..88a3cb2a5 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.codehaus.plexus plexus-compiler-api - 2.9.0 + 2.10.0 From 12e8c0655c968c38966f059e085c5d4a0c4551d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 10:59:12 +0000 Subject: [PATCH 090/856] Bump slf4j-simple from 1.7.35 to 1.7.36 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.35 to 1.7.36. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.35...v_1.7.36) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 88a3cb2a5..538e543b1 100644 --- a/pom.xml +++ b/pom.xml @@ -313,7 +313,7 @@ org.slf4j slf4j-simple - 1.7.35 + 1.7.36 ru.yandex.clickhouse From 5b27bb91dfcc184bcc66fbed0510941684cdc329 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 9 Feb 2022 11:24:26 +0000 Subject: [PATCH 091/856] Bump postgresql from 42.3.1 to 42.3.2 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.1 to 42.3.2. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.1...REL42.3.2) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 538e543b1..d90ed9edd 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.1 + 42.3.2 org.xerial From efb390487cc9a8c6a9dfbc3f0e00c08d88339397 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 09:12:07 +0000 Subject: [PATCH 092/856] Bump maven-javadoc-plugin from 3.3.1 to 3.3.2 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.3.1 to 3.3.2. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.3.1...maven-javadoc-plugin-3.3.2) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d90ed9edd..6c24bbe87 100644 --- a/pom.xml +++ b/pom.xml @@ -397,7 +397,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.3.1 + 3.3.2 8 From 8f1a9ba0a29228a817d35cd92fa571fd76dbfa33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Feb 2022 10:50:32 +0000 Subject: [PATCH 093/856] Bump maven-compiler-plugin from 3.9.0 to 3.10.0 Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.9.0 to 3.10.0. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.9.0...maven-compiler-plugin-3.10.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6c24bbe87..ea5e9b09d 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ maven-compiler-plugin - 3.9.0 + 3.10.0 11 11 From 6390f6fc0f371a233a918702f07afb9e024afeb7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Feb 2022 01:20:38 +0000 Subject: [PATCH 094/856] Bump postgresql from 42.3.2 to 42.3.3 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.2 to 42.3.3. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.2...REL42.3.3) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ea5e9b09d..ff08a32cc 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.2 + 42.3.3 org.xerial From de46775c6d134dd18b800c06f6979bf5b77e1e62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Feb 2022 09:13:30 +0000 Subject: [PATCH 095/856] Bump checkstyle from 9.3 to 10.0 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 9.3 to 10.0. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-9.3...checkstyle-10.0) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ff08a32cc..bf5c9131b 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 9.3 + 10.0 From 961a62f081a9ca4f71a1e56486a11459b3bb7ee1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Mar 2022 09:13:02 +0000 Subject: [PATCH 096/856] Bump formatter-maven-plugin from 2.17.1 to 2.18.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.17.1 to 2.18.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.17.1...formatter-maven-plugin-2.18.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bf5c9131b..0da27bc94 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.17.1 + 2.18.0 eclipseformat From eaab8836e59517decda30a14d0748827c41b9387 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 09:12:42 +0000 Subject: [PATCH 097/856] Bump plexus-compiler-eclipse from 2.10.0 to 2.11.1 Bumps plexus-compiler-eclipse from 2.10.0 to 2.11.1. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0da27bc94..70edee8c6 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.10.0 + 2.11.1 org.eclipse.jdt From a24fc38fa3b224c64a363a43b0c9011c5017c96d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Mar 2022 19:21:13 +0000 Subject: [PATCH 098/856] Bump plexus-compiler-api from 2.10.0 to 2.11.1 Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.10.0 to 2.11.1. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.10.0...plexus-compiler-2.11.1) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 70edee8c6..edfe8edf8 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.codehaus.plexus plexus-compiler-api - 2.10.0 + 2.11.1 From 2b65eff97eed1a6ac21e18ab74093fc46f5ab6de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Mar 2022 09:11:08 +0000 Subject: [PATCH 099/856] Bump maven-compiler-plugin from 3.10.0 to 3.10.1 Bumps [maven-compiler-plugin](https://github.com/apache/maven-compiler-plugin) from 3.10.0 to 3.10.1. - [Release notes](https://github.com/apache/maven-compiler-plugin/releases) - [Commits](https://github.com/apache/maven-compiler-plugin/compare/maven-compiler-plugin-3.10.0...maven-compiler-plugin-3.10.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-compiler-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index edfe8edf8..73cfcf926 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,7 @@ maven-compiler-plugin - 3.10.0 + 3.10.1 11 11 From 60d563cb745e200fce876821765ea4712b2231a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Mar 2022 09:11:43 +0000 Subject: [PATCH 100/856] Bump maven-jxr-plugin from 3.1.1 to 3.2.0 Bumps [maven-jxr-plugin](https://github.com/apache/maven-jxr) from 3.1.1 to 3.2.0. - [Release notes](https://github.com/apache/maven-jxr/releases) - [Commits](https://github.com/apache/maven-jxr/compare/jxr-3.1.1...jxr-3.2.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jxr-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 73cfcf926..207975280 100644 --- a/pom.xml +++ b/pom.xml @@ -341,7 +341,7 @@ org.apache.maven.plugins maven-jxr-plugin - 3.1.1 + 3.2.0 From b6522767fe8bcfa77333cdbd73a36db76f1986f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Mar 2022 09:14:54 +0000 Subject: [PATCH 101/856] Bump spotbugs-maven-plugin from 4.5.3.0 to 4.6.0.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.5.3.0 to 4.6.0.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.5.3.0...spotbugs-maven-plugin-4.6.0.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 207975280..a43bb22c2 100644 --- a/pom.xml +++ b/pom.xml @@ -250,7 +250,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.5.3.0 + 4.6.0.0 spotbugs From 0098bccb5baaac9854e3dc96763743ed555d2ce3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Mar 2022 09:17:57 +0000 Subject: [PATCH 102/856] Bump checkstyle from 10.0 to 10.1 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.0 to 10.1. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.0...checkstyle-10.1) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a43bb22c2..aad66fd4a 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.0 + 10.1 From dcabc89e2fa1dd9619e07a2650cd3d2489268add Mon Sep 17 00:00:00 2001 From: Gavin Chou Date: Mon, 28 Mar 2022 12:16:29 +0800 Subject: [PATCH 103/856] Fix incorrect exit code and possible data race of shutdownThreads * if --max-generated-database is set to positive number and run without error, the exit code should be 0 instead of the one set in MainOptions * shutdownThreads should be atomic due to multi-thread access to it --- src/sqlancer/Main.java | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 77ecdcabe..a4404a6fa 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -17,6 +17,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import com.beust.jcommander.JCommander; @@ -33,7 +34,7 @@ public final class Main { public static volatile AtomicLong nrDatabases = new AtomicLong(); public static volatile AtomicLong nrSuccessfulActions = new AtomicLong(); public static volatile AtomicLong nrUnsuccessfulActions = new AtomicLong(); - static int threadsShutdown; + public static volatile AtomicLong threadsShutdown = new AtomicLong(); static boolean progressMonitorStarted; static { @@ -446,6 +447,7 @@ private String formatInteger(long intValue) { return options.getErrorExitCode(); } } + final AtomicBoolean someOneFails = new AtomicBoolean(false); for (int i = 0; i < options.getTotalNumberTries(); i++) { final String databaseName = options.getDatabasePrefix() + i; @@ -466,23 +468,18 @@ public void run() { private void runThread(final String databaseName) { Randomly r = new Randomly(seed); try { - if (options.getMaxGeneratedDatabases() == -1) { - // run without a limit - boolean continueRunning = true; - while (continueRunning) { - continueRunning = run(options, execService, executorFactory, r, databaseName); - } - } else { - for (int i = 0; i < options.getMaxGeneratedDatabases(); i++) { - boolean continueRunning = run(options, execService, executorFactory, r, databaseName); - if (!continueRunning) { - break; - } + int maxNrDbs = options.getMaxGeneratedDatabases(); + // run without a limit if maxNrDbs == -1 + for (int i = 0; i < maxNrDbs || maxNrDbs == -1; i++) { + Boolean continueRunning = run(options, execService, executorFactory, r, databaseName); + if (!continueRunning) { + someOneFails.set(true); + break; } } } finally { - threadsShutdown++; - if (threadsShutdown == options.getTotalNumberTries()) { + threadsShutdown.addAndGet(1); + if (threadsShutdown.get() == options.getTotalNumberTries()) { execService.shutdown(); } } @@ -527,7 +524,7 @@ private boolean run(MainOptions options, ExecutorService execService, e.printStackTrace(); } - return threadsShutdown == 0 ? 0 : options.getErrorExitCode(); + return someOneFails.get() ? options.getErrorExitCode() : 0; } /** @@ -585,7 +582,7 @@ public void run() { System.out.println(String.format( "[%s] Executed %d queries (%d queries/s; %.2f/s dbs, successful statements: %2d%%). Threads shut down: %d.", dateFormat.format(date), currentNrQueries, (int) throughput, throughputDbs, - successfulStatementsRatio, threadsShutdown)); + successfulStatementsRatio, threadsShutdown.get())); timeMillis = System.currentTimeMillis(); lastNrQueries = currentNrQueries; lastNrDbs = currentNrDbs; From ae302371f6b60c1f016b26276044c5b95f8cd44a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Apr 2022 09:14:53 +0000 Subject: [PATCH 104/856] Bump jacoco-maven-plugin from 0.8.7 to 0.8.8 Bumps [jacoco-maven-plugin](https://github.com/jacoco/jacoco) from 0.8.7 to 0.8.8. - [Release notes](https://github.com/jacoco/jacoco/releases) - [Commits](https://github.com/jacoco/jacoco/compare/v0.8.7...v0.8.8) --- updated-dependencies: - dependency-name: org.jacoco:jacoco-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aad66fd4a..e03babb32 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ org.jacoco jacoco-maven-plugin - 0.8.7 + 0.8.8 From 199903c72f6f47ff1f60236964a39df421822c42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 09:19:58 +0000 Subject: [PATCH 105/856] Bump maven-javadoc-plugin from 3.3.2 to 3.4.0 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.3.2 to 3.4.0. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.3.2...maven-javadoc-plugin-3.4.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e03babb32..fd3cf7fa6 100644 --- a/pom.xml +++ b/pom.xml @@ -397,7 +397,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.3.2 + 3.4.0 8 From 9dccd866a0a12ce300e9c1fc7cf19bd751172a47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:02:12 +0000 Subject: [PATCH 106/856] Bump postgresql from 42.3.3 to 42.3.4 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.3 to 42.3.4. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.3...REL42.3.4) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fd3cf7fa6..d7d42e2d9 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.3 + 42.3.4 org.xerial From 6f3be054db571d4b98b67bbf86ae8b5b6837b528 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 10:16:17 +0000 Subject: [PATCH 107/856] Bump h2 from 2.1.210 to 2.1.212 Bumps [h2](https://github.com/h2database/h2database) from 2.1.210 to 2.1.212. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.210...version-2.1.212) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d7d42e2d9..37c6fdfa5 100644 --- a/pom.xml +++ b/pom.xml @@ -323,7 +323,7 @@ com.h2database h2 - 2.1.210 + 2.1.212 org.mongodb From 9b22e488b7075da6d7e4e71c48aa80e2870f4e76 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 11:55:19 +0000 Subject: [PATCH 108/856] Bump mariadb-java-client from 3.0.3 to 3.0.4 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.3 to 3.0.4. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.3...3.0.4) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 37c6fdfa5..e572c3f32 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.3 + 3.0.4 org.duckdb From 25eea828a412361436df7abd2db47645d9d1c47b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Apr 2022 12:18:32 +0000 Subject: [PATCH 109/856] Bump maven-shade-plugin from 3.2.4 to 3.3.0 Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.2.4 to 3.3.0. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.2.4...maven-shade-plugin-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-shade-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e572c3f32..1ae3092eb 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.2.4 + 3.3.0 package From 6b1e251f0d9cb0c6319a1e51809c90b44c0b1f23 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Apr 2022 09:12:55 +0000 Subject: [PATCH 110/856] Bump maven-antrun-plugin from 3.0.0 to 3.1.0 Bumps [maven-antrun-plugin](https://github.com/apache/maven-antrun-plugin) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/apache/maven-antrun-plugin/releases) - [Commits](https://github.com/apache/maven-antrun-plugin/compare/maven-antrun-plugin-3.0.0...maven-antrun-plugin-3.1.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-antrun-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1ae3092eb..df9da8077 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,7 @@ org.apache.maven.plugins maven-antrun-plugin - 3.0.0 + 3.1.0 package From 36705035fcbe35e41a2bf3b4bb5eebca66120a9e Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Fri, 22 Apr 2022 21:27:08 +0800 Subject: [PATCH 111/856] Add overloaded assumeResultSetsAreEqual which accepts canonicalization function --- src/sqlancer/ComparatorHelper.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index f05fe5ee3..145342ac4 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -6,6 +6,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.UnaryOperator; import java.util.stream.Collectors; import sqlancer.common.query.ExpectedErrors; @@ -114,6 +115,18 @@ public static void assumeResultSetsAreEqual(List resultSet, List } } + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, + String originalQueryString, List combinedString, SQLGlobalState state, + UnaryOperator canonicalizationRule) { + // Overloaded version of assumeResultSetsAreEqual that takes a canonicalization function which is applied to + // both result sets before their comparison. + List canonicalizedResultSet = resultSet.stream().map(canonicalizationRule).collect(Collectors.toList()); + List canonicalizedSecondResultSet = secondResultSet.stream().map(canonicalizationRule) + .collect(Collectors.toList()); + assumeResultSetsAreEqual(canonicalizedResultSet, canonicalizedSecondResultSet, originalQueryString, + combinedString, state); + } + public static List getCombinedResultSet(String firstQueryString, String secondQueryString, String thirdQueryString, List combinedString, boolean asUnion, SQLGlobalState state, ExpectedErrors errors) throws SQLException { From d03e1bde90741de0b522cdfeb6cc4400cd3e2a76 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Fri, 22 Apr 2022 22:32:40 +0800 Subject: [PATCH 112/856] Add tests for assumeResultSetsAreEqual --- test/sqlancer/TestComparatorHelper.java | 54 +++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 test/sqlancer/TestComparatorHelper.java diff --git a/test/sqlancer/TestComparatorHelper.java b/test/sqlancer/TestComparatorHelper.java new file mode 100644 index 000000000..6815b9258 --- /dev/null +++ b/test/sqlancer/TestComparatorHelper.java @@ -0,0 +1,54 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertThrowsExactly; + +import java.util.Arrays; +import java.util.List; + +import org.junit.jupiter.api.Test; + +public class TestComparatorHelper { + // TODO: Implement tests for the other ComparatorHelper methods + + @Test + public void testAssumeResultSetsAreEqualWithEqualSets() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "c"); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + + } + + @Test + public void testAssumeResultSetsAreEqualWithUnequalLengthSets() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "c", "d", "g"); + // NullPointerException is raised instead of AssertionError because state is null and the state.getState()... + // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions + // is raised + assertThrowsExactly(NullPointerException.class, () -> { + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + }); + } + + @Test + public void testAssumeResultSetsAreEqualWithUnequalValueSets() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "d"); + // NullPointerException is raised instead of AssertionError because state is null and the state.getState()... + // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions + // is raised + assertThrowsExactly(NullPointerException.class, () -> { + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + }); + } + + @Test + public void testAssumeResultSetsAreEqualWithCanonicalizationRule() { + List r1 = Arrays.asList("a", "b", "c"); + List r2 = Arrays.asList("a", "b", "d"); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null, (String s) -> { + return s.equals("d") ? "c" : s; + }); + } + +} From 5c5555af385b08004f033df2eedfb5504e736705 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 18 Apr 2022 15:41:17 +0800 Subject: [PATCH 113/856] Update duckdb implementation to that of hannes/sqlancer@persistent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hannes Mühleisen Co-authored-by: Mark Raasveldt Co-authored-by: Richard Wesley --- src/sqlancer/duckdb/DuckDBErrors.java | 28 +++++++++++----- src/sqlancer/duckdb/DuckDBProvider.java | 32 +++++++++++++++++-- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- .../duckdb/gen/DuckDBExpressionGenerator.java | 5 ++- .../duckdb/gen/DuckDBTableGenerator.java | 1 - 5 files changed, 52 insertions(+), 16 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 0cd9f52e3..cb45bb96c 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -23,7 +23,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("invalid UTF-8"); // TODO errors.add("String value is not valid UTF8"); - errors.add("Conversion: Invalid TypeId "); // TODO + errors.add("Invalid TypeId "); // TODO errors.add("GROUP BY clause cannot contain aggregates!"); // investigate @@ -33,7 +33,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Overflow in multiplication"); errors.add("Out of Range"); - errors.add("Conversion: Date out of range"); + errors.add("Date out of range"); // collate errors.add("Cannot combine types with different collation!"); @@ -42,6 +42,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { // // https://github.com/cwida/duckdb/issues/532 errors.add("Not implemented type: DATE"); errors.add("Not implemented type: TIMESTAMP"); + errors.add("Like pattern must not end with escape character!"); // LIKE errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a // table supports rowids @@ -50,7 +51,6 @@ public static void addExpressionErrors(ExpectedErrors errors) { // removed column errors.add("Contents of view were altered: types don't match!"); errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); - errors.add("Could not convert string"); // Could not convert string '1.7976931348623157E308' to DOUBLE } private static void addRegexErrors(ExpectedErrors errors) { @@ -66,32 +66,44 @@ private static void addRegexErrors(ExpectedErrors errors) { } private static void addFunctionErrors(ExpectedErrors errors) { - errors.add("SUBSTRING cannot handle negative offsets"); + errors.add("SUBSTRING cannot handle negative lengths"); errors.add("is undefined outside [-1,1]"); // ACOS etc errors.add("invalid type specifier"); // PRINTF errors.add("argument index out of range"); // PRINTF errors.add("invalid format string"); // PRINTF errors.add("number is too big"); // PRINTF + errors.add("Like pattern must not end with escape character!"); // LIKE errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part errors.add("extract specifier"); // date_part + errors.add("not recognized"); // date_part + errors.add("not supported"); // date_part + errors.add("Failed to cast"); + errors.add("Conversion Error"); + errors.add("Could not cast value"); errors.add("Insufficient padding in RPAD"); // RPAD errors.add("Could not choose a best candidate function for the function call"); // monthname + errors.add("expected a numeric precision field"); // ROUND + errors.add("with non-constant precision is not supported"); // ROUND } public static void addInsertErrors(ExpectedErrors errors) { + addRegexErrors(errors); + addFunctionErrors(errors); + errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); - errors.add("duplicate key value violates primary key or unique constraint"); + errors.add("duplicate key"); errors.add("can't be cast because the value is out of range for the destination type"); errors.add("Could not convert string"); - errors.add("timestamp field value out of range"); errors.add("Unimplemented type for cast"); - errors.add("date/time field value out of range"); + errors.add("field value out of range"); errors.add("CHECK constraint failed"); errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if // the table has a primary key - errors.add("Out of Range: Could not cast value"); + errors.add("Could not cast value"); + errors.add("create unique index, table contains duplicate data"); + errors.add("Failed to cast"); } public static void addGroupByErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 8453bb6f7..5be773885 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -2,6 +2,9 @@ import java.sql.DriverManager; import java.sql.SQLException; +import java.sql.Connection; +import java.sql.Statement; +import java.io.File; import com.google.auto.service.AutoService; @@ -118,11 +121,34 @@ public void generateDatabase(DuckDBGlobalState globalState) throws Exception { se.executeStatements(); } + public void tryDeleteFile(String fname) { + try { + File f = new File(fname); + f.delete(); + } catch(Exception e) { + } + } + + public void tryDeleteDatabase(String dbpath) { + if (dbpath.equals("") || dbpath.equals(":memory:")) { + return; + } + tryDeleteFile(dbpath); + tryDeleteFile(dbpath + ".wal"); + } + @Override public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLException { - String url = "jdbc:duckdb:"; - return new SQLConnection(DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword())); + String database_file = System.getProperty("duckdb.database.file", ""); + String url = "jdbc:duckdb:" + database_file; + tryDeleteDatabase(database_file); + + Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), + globalState.getOptions().getPassword()); + Statement stmt = conn.createStatement(); + stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); + stmt.close(); + return new SQLConnection(conn); } @Override diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 56e89686c..60395976c 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -237,7 +237,7 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master()")) { + try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master WHERE type='table' or type='view'")) { while (rs.next()) { tableNames.add(rs.getString("name")); } diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index f907e62f2..fc9ae1b2b 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -243,6 +243,7 @@ public enum DBFunction { DEGREES(1), // RADIANS(1), // MOD(2), // + XOR(2), // // string functions LENGTH(1), // LOWER(1), // @@ -379,8 +380,7 @@ public static Operator getRandom() { } public enum DuckDBBinaryArithmeticOperator implements Operator { - CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), XOR("#"), LSHIFT("<<"), - RSHIFT(">>"); + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); private String textRepr; @@ -400,7 +400,6 @@ public String getTextRepresentation() { } public enum DuckDBBinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index bb8fe2304..4d2365f07 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -58,7 +58,6 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); - DuckDBErrors.addExpressionErrors(errors); sb.append(")"); } } From 6dd5d531c40ab5c57ab76160866b5a019b5da5a6 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 18 Apr 2022 20:38:46 +0800 Subject: [PATCH 114/856] Bump duckdb_jdbc from 0.2.5 to 0.3.3 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index df9da8077..aa39f5f2b 100644 --- a/pom.xml +++ b/pom.xml @@ -302,7 +302,7 @@ org.duckdb duckdb_jdbc - 0.2.5 + 0.3.3 org.junit.jupiter From e6a2918c153cdf530832d6d4825a276fd62b0523 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 18 Apr 2022 20:39:52 +0800 Subject: [PATCH 115/856] Format updated implementation --- src/sqlancer/duckdb/DuckDBProvider.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 5be773885..3164b7d6a 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -1,10 +1,10 @@ package sqlancer.duckdb; +import java.io.File; +import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; -import java.sql.Connection; import java.sql.Statement; -import java.io.File; import com.google.auto.service.AutoService; @@ -125,7 +125,7 @@ public void tryDeleteFile(String fname) { try { File f = new File(fname); f.delete(); - } catch(Exception e) { + } catch (Exception e) { } } @@ -139,9 +139,9 @@ public void tryDeleteDatabase(String dbpath) { @Override public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLException { - String database_file = System.getProperty("duckdb.database.file", ""); - String url = "jdbc:duckdb:" + database_file; - tryDeleteDatabase(database_file); + String databaseFile = System.getProperty("duckdb.database.file", ""); + String url = "jdbc:duckdb:" + databaseFile; + tryDeleteDatabase(databaseFile); Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); From f43a0bebc8ae30b2da38d95d9e88f028dae3d606 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 18 Apr 2022 20:42:54 +0800 Subject: [PATCH 116/856] Re-enable QUERY_PARTITIONING oracle in CI for DuckDB --- test/sqlancer/dbms/TestDuckDB.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDB.java index a2f3e89be..d36d8610f 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDB.java @@ -11,8 +11,10 @@ public class TestDuckDB { @Test public void testDuckDB() { // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 - assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", "0", "duckdb", "--oracle", "QUERY_PARTITIONING" })); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "duckdb", "--oracle", + "QUERY_PARTITIONING" })); } } From 4a5724e8eb0d162987f32683f05e16c8feb3ce2c Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Sat, 23 Apr 2022 07:36:04 +0800 Subject: [PATCH 117/856] Add canonicalization rule for DuckDB results --- .../duckdb/test/DuckDBQueryPartitioningBase.java | 10 ++++++++++ .../test/DuckDBQueryPartitioningDistinctTester.java | 2 +- .../test/DuckDBQueryPartitioningGroupByTester.java | 2 +- .../test/DuckDBQueryPartitioningHavingTester.java | 2 +- .../test/DuckDBQueryPartitioningWhereTester.java | 2 +- 5 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index a8663b314..08c862973 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -36,6 +37,15 @@ public DuckDBQueryPartitioningBase(DuckDBGlobalState state) { DuckDBErrors.addExpressionErrors(errors); } + public static String canonicalizeResultValue(String value) { + // Rule: -0.0 should be canonicalized to 0.0 + if (Objects.equals(value, "-0.0")) { + return "0.0"; + } + + return value; + } + @Override public void check() throws SQLException { s = state.getSchema(); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java index 89b7ea602..d85d2a2d5 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java @@ -38,7 +38,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java index 7ac650f44..fab480fed 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java @@ -41,7 +41,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index e11dcefb3..fc8843729 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -46,7 +46,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java index 132c02a62..412ab6ffd 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java @@ -39,7 +39,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); } } From 6af2b84bb809b35bd64116302682a09c664f307c Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 25 Apr 2022 18:00:21 +0800 Subject: [PATCH 118/856] Add support for NULL data type in DuckDB --- src/sqlancer/duckdb/DuckDBSchema.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 60395976c..8577fff26 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -23,7 +23,7 @@ public class DuckDBSchema extends AbstractSchema public enum DuckDBDataType { - INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP; + INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP, NULL; public static DuckDBDataType getRandom() { return Randomly.fromOptions(values()); @@ -109,6 +109,8 @@ public String toString() { return Randomly.fromOptions("TIMESTAMP", "DATETIME"); case DATE: return Randomly.fromOptions("DATE"); + case NULL: + return Randomly.fromOptions("NULL"); default: throw new AssertionError(getPrimitiveDataType()); } @@ -197,6 +199,9 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { case "TIMESTAMP": primitiveType = DuckDBDataType.TIMESTAMP; break; + case "NULL": + primitiveType = DuckDBDataType.NULL; + break; case "INTERVAL": throw new IgnoreMeException(); // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP From 09dff0bc13a1d2b06a240497709af46e187a8c44 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Mon, 25 Apr 2022 18:02:32 +0800 Subject: [PATCH 119/856] Exclude NULL from DuckDB data type randomization, rename for clarity --- src/sqlancer/duckdb/DuckDBSchema.java | 12 ++++++++---- .../duckdb/gen/DuckDBAlterTableGenerator.java | 4 ++-- .../duckdb/gen/DuckDBExpressionGenerator.java | 5 +++-- src/sqlancer/duckdb/gen/DuckDBTableGenerator.java | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 8577fff26..6b94ce137 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -25,8 +25,12 @@ public enum DuckDBDataType { INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP, NULL; - public static DuckDBDataType getRandom() { - return Randomly.fromOptions(values()); + public static DuckDBDataType getRandomWithoutNull() { + DuckDBDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DuckDBDataType.NULL); + return dt; } } @@ -53,8 +57,8 @@ public int getSize() { return size; } - public static DuckDBCompositeDataType getRandom() { - DuckDBDataType type = DuckDBDataType.getRandom(); + public static DuckDBCompositeDataType getRandomWithoutNull() { + DuckDBDataType type = DuckDBDataType.getRandomWithoutNull(); int size = -1; switch (type) { case INT: diff --git a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java index 577ad566b..710fca1e2 100644 --- a/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBAlterTableGenerator.java @@ -34,13 +34,13 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { String columnName = table.getFreeColumnName(); sb.append(columnName); sb.append(" "); - sb.append(DuckDBCompositeDataType.getRandom().toString()); + sb.append(DuckDBCompositeDataType.getRandomWithoutNull().toString()); break; case ALTER_COLUMN: sb.append("ALTER COLUMN "); sb.append(table.getRandomColumn().getName()); sb.append(" SET DATA TYPE "); - sb.append(DuckDBCompositeDataType.getRandom().toString()); + sb.append(DuckDBCompositeDataType.getRandomWithoutNull().toString()); if (Randomly.getBoolean()) { sb.append(" USING "); DuckDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index fc9ae1b2b..e0270e816 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -98,7 +98,8 @@ protected Node generateExpression(int depth) { return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), DuckDBBinaryArithmeticOperator.getRandom()); case CAST: - return new DuckDBCastOperation(generateExpression(depth + 1), DuckDBCompositeDataType.getRandom()); + return new DuckDBCastOperation(generateExpression(depth + 1), + DuckDBCompositeDataType.getRandomWithoutNull()); case FUNC: DBFunction func = DBFunction.getRandom(); return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); @@ -132,7 +133,7 @@ public Node generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return DuckDBConstant.createNullConstant(); } - DuckDBDataType type = DuckDBDataType.getRandom(); + DuckDBDataType type = DuckDBDataType.getRandomWithoutNull(); switch (type) { case INT: if (!globalState.getDbmsSpecificOptions().testIntConstants) { diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index 4d2365f07..d3cd27fde 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -80,7 +80,7 @@ private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { String columnName = String.format("c%d", i); - DuckDBCompositeDataType columnType = DuckDBCompositeDataType.getRandom(); + DuckDBCompositeDataType columnType = DuckDBCompositeDataType.getRandomWithoutNull(); columns.add(new DuckDBColumn(columnName, columnType, false, false)); } return columns; From 27d178806918ffaa216d1a0498c7e7489476b243 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Apr 2022 10:16:25 +0000 Subject: [PATCH 120/856] Bump checkstyle from 10.1 to 10.2 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.1 to 10.2. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.1...checkstyle-10.2) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa39f5f2b..ceaa5cfb4 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.1 + 10.2 From abe47f30af704dad7d8f8f4bb679d10497a09bea Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Fri, 29 Apr 2022 13:07:20 +0800 Subject: [PATCH 121/856] fix bug in the copy constructor of SQLite3Select --- src/sqlancer/sqlite3/ast/SQLite3Select.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 340c708fb..bff7fb775 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -30,7 +30,7 @@ public SQLite3Select(SQLite3Select other) { limitClause = other.limitClause; orderByClause = new ArrayList<>(other.orderByClause); offsetClause = other.offsetClause; - fetchColumns = new ArrayList<>(fetchColumns); + fetchColumns = new ArrayList<>(other.fetchColumns); joinStatements = new ArrayList<>(); for (Join j : other.joinStatements) { joinStatements.add(new Join(j)); From 443ba9224952da4592043ed9f5b649c77d856948 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 09:12:19 +0000 Subject: [PATCH 122/856] Bump plexus-compiler-eclipse from 2.11.1 to 2.12.0 Bumps plexus-compiler-eclipse from 2.11.1 to 2.12.0. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ceaa5cfb4..4de9b0ad1 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.11.1 + 2.12.0 org.eclipse.jdt From ed5d897ff244358faa7be049f6a2e481d0533cc0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 May 2022 11:54:01 +0000 Subject: [PATCH 123/856] Bump plexus-compiler-api from 2.11.1 to 2.12.0 Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.11.1 to 2.12.0. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.11.1...plexus-compiler-2.12.0) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4de9b0ad1..75247e015 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.codehaus.plexus plexus-compiler-api - 2.11.1 + 2.12.0 From fa889dedba13caac97bd76fceb133de13bb9aaca Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 10 May 2022 10:28:28 +0800 Subject: [PATCH 124/856] fix invalid trigger generation --- src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java index 610385501..1e5254948 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateTriggerGenerator.java @@ -100,11 +100,11 @@ public static SQLQueryAdapter create(SQLite3GlobalState globalState) throws SQLE sb.append(", 'asdf'"); } sb.append(")"); + sb.append(";"); break; default: throw new AssertionError(); } - sb.append(";"); } sb.append("END"); From f1800b099a5d366ae26314cbfdef7e002e3c2ee2 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: Tue, 10 May 2022 21:14:40 +0500 Subject: [PATCH 125/856] Add '--extensions' option. Allow to perform a sqlancer test with an relocatable extension installed. --- src/sqlancer/postgres/PostgresOptions.java | 3 +++ src/sqlancer/postgres/PostgresProvider.java | 16 ++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index ed2856451..b6c79d752 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -38,6 +38,9 @@ public class PostgresOptions implements DBMSSpecificOptions { NOREC { @Override diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 0aeb29328..e9396a628 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -63,6 +63,7 @@ public class PostgresProvider extends SQLProviderAdapter Date: Tue, 17 May 2022 13:02:50 +0000 Subject: [PATCH 126/856] Deterministic Hex and Dec for SQLite3 --- .../sqlite3/SQLite3ToStringVisitor.java | 6 +++--- src/sqlancer/sqlite3/ast/SQLite3Constant.java | 20 +++++++++++++++++++ .../gen/SQLite3ExpressionGenerator.java | 2 +- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 4d0820b25..f9f96a1d3 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -177,11 +177,11 @@ public void visit(SQLite3Constant c) { // sb.append(c.asInt() == 1 ? "TRUE" : "FALSE"); // } else { // - 0X8000000000000000 results in an error message otherwise - if (Randomly.getBoolean() || c.asInt() == Long.MIN_VALUE) { - sb.append(c.asInt()); - } else { + if (c.isHex()) { long intVal = c.asInt(); asHexString(intVal); + } else { + sb.append(c.asInt()); } // } break; diff --git a/src/sqlancer/sqlite3/ast/SQLite3Constant.java b/src/sqlancer/sqlite3/ast/SQLite3Constant.java index abe3cd033..a6edd7bf9 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Constant.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Constant.java @@ -63,9 +63,21 @@ public SQLite3Constant applyLess(SQLite3Constant right, SQLite3CollateSequence c public static class SQLite3IntConstant extends SQLite3Constant { private final long value; + private final boolean isHex; + + public SQLite3IntConstant(long value, boolean isHex) { + this.value = value; + this.isHex = isHex; + } public SQLite3IntConstant(long value) { this.value = value; + this.isHex = false; + } + + @Override + public boolean isHex() { + return isHex; } @Override @@ -511,6 +523,10 @@ public SQLite3Constant applyLess(SQLite3Constant right, SQLite3CollateSequence c public abstract Object getValue(); + public boolean isHex() { + throw new UnsupportedOperationException(this.getDataType().toString()); + } + public long asInt() { throw new UnsupportedOperationException(this.getDataType().toString()); } @@ -533,6 +549,10 @@ public static SQLite3Constant createIntConstant(long val) { return new SQLite3IntConstant(val); } + public static SQLite3Constant createIntConstant(long val, boolean isHex) { + return new SQLite3IntConstant(val, isHex); + } + public static SQLite3Constant createBinaryConstant(byte[] val) { return new SQLite3BinaryConstant(val); } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index e7480dc19..d21f00af3 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -185,7 +185,7 @@ private SQLite3Expression getRandomLiteralValueInternal(Randomly r) { switch (randomLiteral) { case INTEGER: if (Randomly.getBoolean()) { - return SQLite3Constant.createIntConstant(r.getInteger()); + return SQLite3Constant.createIntConstant(r.getInteger(), Randomly.getBoolean()); } else { return SQLite3Constant.createTextConstant(String.valueOf(r.getInteger())); } From 1da51fb1542c0fba292d9ba39b50cb3bd2010ab1 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 17 May 2022 13:25:44 +0000 Subject: [PATCH 127/856] fix too large hex bug in SQLite3 --- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index f9f96a1d3..f81259732 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -177,11 +177,11 @@ public void visit(SQLite3Constant c) { // sb.append(c.asInt() == 1 ? "TRUE" : "FALSE"); // } else { // - 0X8000000000000000 results in an error message otherwise - if (c.isHex()) { + if (!c.isHex() || c.asInt() == Long.MIN_VALUE) { + sb.append(c.asInt()); + } else { long intVal = c.asInt(); asHexString(intVal); - } else { - sb.append(c.asInt()); } // } break; From 57724bc433be2b13a7b0f1647487ef2fc11bbcb1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 May 2022 09:13:35 +0000 Subject: [PATCH 128/856] Bump spotbugs-maven-plugin from 4.6.0.0 to 4.7.0.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.6.0.0 to 4.7.0.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.6.0.0...spotbugs-maven-plugin-4.7.0.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 75247e015..3f251a9a1 100644 --- a/pom.xml +++ b/pom.xml @@ -250,7 +250,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.6.0.0 + 4.7.0.0 spotbugs From 6d7490afb828a12c6fd2aef170bf8fe4eb38fb2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 May 2022 09:31:53 +0000 Subject: [PATCH 129/856] Bump postgresql from 42.3.4 to 42.3.5 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.4 to 42.3.5. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.4...REL42.3.5) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3f251a9a1..d6f229e24 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.4 + 42.3.5 org.xerial From ea4d2a0767233453012a2a5e49839c5d7e5fec83 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: Sat, 28 May 2022 22:17:55 +0300 Subject: [PATCH 130/856] Add an expected error on usage of the chr() routine. According to PG15 commit e9e63b7, chr() in next release will return an error on negative input. Be prepared to that change. --- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 771fc8651..a3e04d046 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -63,6 +63,7 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("is of type boolean but expression is of type text"); errors.add("a negative number raised to a non-integer power yields a complex result"); errors.add("could not determine polymorphic type because input has type unknown"); + errors.add("character number must be positive"); addToCharFunctionErrors(errors); addBitStringOperationErrors(errors); addFunctionErrors(errors); From b2c504675d39164dc28a604bba9380b0a70f909d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 29 May 2022 09:26:56 +0000 Subject: [PATCH 131/856] Bump mariadb-java-client from 3.0.4 to 3.0.5 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.4 to 3.0.5. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.4...3.0.5) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d6f229e24..d1b9e3660 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.4 + 3.0.5 org.duckdb From 3386cb2ad002e0664d72abf22b8e8b64bf0ceb38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 29 May 2022 09:39:39 +0000 Subject: [PATCH 132/856] Bump postgresql from 42.3.5 to 42.3.6 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.5 to 42.3.6. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.5...REL42.3.6) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d1b9e3660..096779d86 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.5 + 42.3.6 org.xerial From a88dcad000ac626e03bb1f6f7f3a0cb56c31d8a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 May 2022 09:15:23 +0000 Subject: [PATCH 133/856] Bump checkstyle from 10.2 to 10.3 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.2 to 10.3. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.2...checkstyle-10.3) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 096779d86..01b84368a 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.2 + 10.3 From 48548200dc90d6c02824f54b4836b2b919c58473 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 May 2022 09:30:43 +0000 Subject: [PATCH 134/856] Bump mysql-connector-java from 8.0.28 to 8.0.29 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.28 to 8.0.29. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.28...8.0.29) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 01b84368a..409521b23 100644 --- a/pom.xml +++ b/pom.xml @@ -292,7 +292,7 @@ mysql mysql-connector-java - 8.0.28 + 8.0.29 org.mariadb.jdbc From e9472486476cf9e62f09b557d12327cdb0608762 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 2 Jun 2022 09:12:19 +0000 Subject: [PATCH 135/856] Bump formatter-maven-plugin from 2.18.0 to 2.19.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.18.0 to 2.19.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.18.0...formatter-maven-plugin-2.19.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 409521b23..aa42765d6 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.18.0 + 2.19.0 eclipseformat From 9ad4cf4381ec294e79849c3349e56eb2040cc97d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jun 2022 09:13:48 +0000 Subject: [PATCH 136/856] Bump postgresql from 42.3.6 to 42.4.0 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.3.6 to 42.4.0. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.3.6...REL42.4.0) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index aa42765d6..2273d6762 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.3.6 + 42.4.0 org.xerial From d41212d665fa4398829937d0ce59fffd03c74614 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jun 2022 09:16:58 +0000 Subject: [PATCH 137/856] Bump h2 from 2.1.212 to 2.1.214 Bumps [h2](https://github.com/h2database/h2database) from 2.1.212 to 2.1.214. - [Release notes](https://github.com/h2database/h2database/releases) - [Commits](https://github.com/h2database/h2database/compare/version-2.1.212...version-2.1.214) --- updated-dependencies: - dependency-name: com.h2database:h2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2273d6762..f43896a43 100644 --- a/pom.xml +++ b/pom.xml @@ -323,7 +323,7 @@ com.h2database h2 - 2.1.212 + 2.1.214 org.mongodb From fbc3fce132bfe0c567b5d3e55a4cb1c474206a06 Mon Sep 17 00:00:00 2001 From: Kok Rui Date: Thu, 16 Jun 2022 11:50:49 +0800 Subject: [PATCH 138/856] Swap parameters of Untyped generateExpressions to (nr, depth) --- .../arangodb/gen/ArangoDBComputedExpressionGenerator.java | 2 +- src/sqlancer/common/gen/UntypedExpressionGenerator.java | 2 +- src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java | 6 +++--- src/sqlancer/h2/H2ExpressionGenerator.java | 4 ++-- .../mongodb/gen/MongoDBComputedExpressionGenerator.java | 2 +- src/sqlancer/tidb/TiDBExpressionGenerator.java | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java index 8a3b98871..01e2e557a 100644 --- a/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java +++ b/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java @@ -65,7 +65,7 @@ protected Node generateExpression(int depth) { return generateLeafNode(); } ComputedFunction function = ComputedFunction.getRandom(); - return new NewFunctionNode<>(generateExpressions(depth + 1, function.getNrArgs()), function); + return new NewFunctionNode<>(generateExpressions(function.getNrArgs(), depth + 1), function); } @Override diff --git a/src/sqlancer/common/gen/UntypedExpressionGenerator.java b/src/sqlancer/common/gen/UntypedExpressionGenerator.java index b52989624..5a244b53b 100644 --- a/src/sqlancer/common/gen/UntypedExpressionGenerator.java +++ b/src/sqlancer/common/gen/UntypedExpressionGenerator.java @@ -42,7 +42,7 @@ public List generateExpressions(int nr) { return expressions; } - public List generateExpressions(int depth, int nr) { + public List generateExpressions(int nr, int depth) { List expressions = new ArrayList<>(); for (int i = 0; i < nr; i++) { expressions.add(generateExpression(depth)); diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index e0270e816..eeba94027 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -48,7 +48,7 @@ protected Node generateExpression(int depth) { if (allowAggregates && Randomly.getBoolean()) { DuckDBAggregateFunction aggregate = DuckDBAggregateFunction.getRandom(); allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); + return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); if (!globalState.getDbmsSpecificOptions().testCollate) { @@ -108,11 +108,11 @@ protected Node generateExpression(int depth) { generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); case IN: return new NewInOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, nr), generateExpressions(depth + 1, nr), + generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), generateExpression(depth + 1)); case LIKE_ESCAPE: return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index b60529ad7..d04251409 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -53,14 +53,14 @@ protected Node generateExpression(int depth) { H2UnaryPrefixOperator.getRandom()); case IN: return new NewInOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case BETWEEN: return new NewBetweenOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(depth + 1, nr), generateExpressions(depth + 1, nr), + generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), generateExpression(depth + 1)); case BINARY_ARITHMETIC: return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), diff --git a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java index fd5959ce6..347d5bc97 100644 --- a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java +++ b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java @@ -33,7 +33,7 @@ protected Node generateExpression(int depth) { return generateLeafNode(); } ComputedFunction func = ComputedFunction.getRandom(); - return new NewFunctionNode<>(generateExpressions(depth + 1, func.getNrArgs()), func); + return new NewFunctionNode<>(generateExpressions(func.getNrArgs(), depth + 1), func); } public MongoDBComputedExpressionGenerator(MongoDBGlobalState globalState) { diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index dc378cc42..a960363fe 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -91,7 +91,7 @@ protected TiDBExpression generateExpression(int depth) { Randomly.fromOptions("utf8mb4_bin", "latin1_bin", "binary", "ascii_bin", "utf8_bin")); case FUNCTION: TiDBFunction func = TiDBFunction.getRandom(); - return new TiDBFunctionCall(func, generateExpressions(depth, func.getNrArgs())); + return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); case BINARY_BIT: return new TiDBBinaryBitOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryBitOperator.getRandom()); @@ -117,8 +117,8 @@ protected TiDBExpression generateExpression(int depth) { throw new IgnoreMeException(); } int nr = Randomly.fromOptions(1, 2); - return new TiDBCase(generateExpression(depth + 1), generateExpressions(depth + 1, nr), - generateExpressions(depth + 1, nr), generateExpression(depth + 1)); + return new TiDBCase(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); default: throw new AssertionError(); } From fc900a6ae389ff57f9a74c67e5a7381447d91956 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 21 Jun 2022 09:11:20 +0000 Subject: [PATCH 139/856] Bump duckdb_jdbc from 0.3.3 to 0.4.0 Bumps [duckdb_jdbc](https://github.com/duckdb/duckdb) from 0.3.3 to 0.4.0. - [Release notes](https://github.com/duckdb/duckdb/releases) - [Commits](https://github.com/duckdb/duckdb/compare/v0.3.3...v0.4.0) --- updated-dependencies: - dependency-name: org.duckdb:duckdb_jdbc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f43896a43..e09cc0954 100644 --- a/pom.xml +++ b/pom.xml @@ -302,7 +302,7 @@ org.duckdb duckdb_jdbc - 0.3.3 + 0.4.0 org.junit.jupiter From 718c93307441c2e08b55e26226de59ab03c2b48d Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 21 Jun 2022 22:08:52 +0800 Subject: [PATCH 140/856] Fix TiDB connection bug --- src/sqlancer/tidb/TiDBProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index f0d8f261b..af41fb143 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -162,8 +162,8 @@ public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLExcep s.execute(createDatabaseCommand); } con.close(); - con = DriverManager.getConnection("jdbc:mysql://127.0.0.1:4000/" + databaseName, - globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); + con = DriverManager.getConnection(url + databaseName, globalState.getOptions().getUserName(), + globalState.getOptions().getPassword()); return new SQLConnection(con); } From 5636a1e560227362af5d674459d23ab58437e407 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Tue, 28 Jun 2022 11:37:11 +0800 Subject: [PATCH 141/856] from duckdb copy and change to databend --- pom.xml | 2 + src/sqlancer/databend/DatabendErrors.java | 114 +++++ src/sqlancer/databend/DatabendOptions.java | 151 ++++++ src/sqlancer/databend/DatabendProvider.java | 154 +++++++ src/sqlancer/databend/DatabendSchema.java | 277 +++++++++++ .../databend/DatabendToStringVisitor.java | 91 ++++ .../databend/ast/DatabendConstant.java | 192 ++++++++ .../databend/ast/DatabendExpression.java | 5 + src/sqlancer/databend/ast/DatabendJoin.java | 124 +++++ src/sqlancer/databend/ast/DatabendSelect.java | 18 + .../gen/DatabendAlterTableGenerator.java | 70 +++ .../databend/gen/DatabendDeleteGenerator.java | 30 ++ .../gen/DatabendExpressionGenerator.java | 436 ++++++++++++++++++ .../databend/gen/DatabendIndexGenerator.java | 59 +++ .../databend/gen/DatabendInsertGenerator.java | 53 +++ .../gen/DatabendRandomQuerySynthesizer.java | 70 +++ .../databend/gen/DatabendTableGenerator.java | 89 ++++ .../databend/gen/DatabendUpdateGenerator.java | 48 ++ .../databend/gen/DatabendViewGenerator.java | 37 ++ .../databend/test/DatabendNoRECOracle.java | 133 ++++++ ...abendQueryPartitioningAggregateTester.java | 188 ++++++++ .../test/DatabendQueryPartitioningBase.java | 82 ++++ ...tabendQueryPartitioningDistinctTester.java | 44 ++ ...atabendQueryPartitioningGroupByTester.java | 53 +++ ...DatabendQueryPartitioningHavingTester.java | 62 +++ .../DatabendQueryPartitioningWhereTester.java | 45 ++ test/sqlancer/dbms/TestDatabend.java | 18 + 27 files changed, 2645 insertions(+) create mode 100644 src/sqlancer/databend/DatabendErrors.java create mode 100644 src/sqlancer/databend/DatabendOptions.java create mode 100644 src/sqlancer/databend/DatabendProvider.java create mode 100644 src/sqlancer/databend/DatabendSchema.java create mode 100644 src/sqlancer/databend/DatabendToStringVisitor.java create mode 100644 src/sqlancer/databend/ast/DatabendConstant.java create mode 100644 src/sqlancer/databend/ast/DatabendExpression.java create mode 100644 src/sqlancer/databend/ast/DatabendJoin.java create mode 100644 src/sqlancer/databend/ast/DatabendSelect.java create mode 100644 src/sqlancer/databend/gen/DatabendAlterTableGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendDeleteGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendExpressionGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendIndexGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendInsertGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java create mode 100644 src/sqlancer/databend/gen/DatabendTableGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendUpdateGenerator.java create mode 100644 src/sqlancer/databend/gen/DatabendViewGenerator.java create mode 100644 src/sqlancer/databend/test/DatabendNoRECOracle.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningBase.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java create mode 100644 src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java create mode 100644 test/sqlancer/dbms/TestDatabend.java diff --git a/pom.xml b/pom.xml index e09cc0954..0cdbf30b4 100644 --- a/pom.xml +++ b/pom.xml @@ -106,6 +106,7 @@ + org.apache.maven.plugins maven-compiler-plugin 3.10.1 @@ -153,6 +154,7 @@ org.apache.maven.plugins maven-dependency-plugin + 3.2.0 copy-dependencies diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java new file mode 100644 index 000000000..92c7b17f7 --- /dev/null +++ b/src/sqlancer/databend/DatabendErrors.java @@ -0,0 +1,114 @@ +package sqlancer.databend; + +import sqlancer.common.query.ExpectedErrors; + +public final class DatabendErrors { + + private DatabendErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.add("with non-constant precision is not supported"); + errors.add("Like pattern must not end with escape character"); + errors.add("Could not convert string"); + errors.add("ORDER term out of range - should be between "); + errors.add("You might need to add explicit type casts."); + errors.add("can't be cast because the value is out of range for the destination type"); + errors.add("field value out of range"); + errors.add("Unimplemented type for cast"); + + errors.add("Escape string must be empty or one character."); + errors.add("Type mismatch when combining rows"); // BETWEEN + + errors.add("invalid UTF-8"); // TODO + errors.add("String value is not valid UTF8"); + + errors.add("Invalid TypeId "); // TODO + + errors.add("GROUP BY clause cannot contain aggregates!"); // investigate + + addRegexErrors(errors); + + addFunctionErrors(errors); + + errors.add("Overflow in multiplication"); + errors.add("Out of Range"); + errors.add("Date out of range"); + + // collate + errors.add("Cannot combine types with different collation!"); + errors.add("collations are only supported for type varchar"); + + // // https://github.com/cwida/Databend/issues/532 + errors.add("Not implemented type: DATE"); + errors.add("Not implemented type: TIMESTAMP"); + errors.add("Like pattern must not end with escape character!"); // LIKE + + errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a + // table supports rowids + + errors.add("does not have a column named"); // TODO: this only happens for views whose underlying table has a + // removed column + errors.add("Contents of view were altered: types don't match!"); + errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + } + + private static void addRegexErrors(ExpectedErrors errors) { + errors.add("missing ]"); + errors.add("missing )"); + errors.add("invalid escape sequence"); + errors.add("no argument for repetition operator: "); + errors.add("bad repetition operator"); + errors.add("trailing \\"); + errors.add("invalid perl operator"); + errors.add("invalid character class range"); + errors.add("width is not integer"); + } + + private static void addFunctionErrors(ExpectedErrors errors) { + errors.add("SUBSTRING cannot handle negative lengths"); + errors.add("is undefined outside [-1,1]"); // ACOS etc + errors.add("invalid type specifier"); // PRINTF + errors.add("argument index out of range"); // PRINTF + errors.add("invalid format string"); // PRINTF + errors.add("number is too big"); // PRINTF + errors.add("Like pattern must not end with escape character!"); // LIKE + errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part + errors.add("extract specifier"); // date_part + errors.add("not recognized"); // date_part + errors.add("not supported"); // date_part + errors.add("Failed to cast"); + errors.add("Conversion Error"); + errors.add("Could not cast value"); + errors.add("Insufficient padding in RPAD"); // RPAD + errors.add("Could not choose a best candidate function for the function call"); // monthname + errors.add("expected a numeric precision field"); // ROUND + errors.add("with non-constant precision is not supported"); // ROUND + } + + public static void addInsertErrors(ExpectedErrors errors) { + addRegexErrors(errors); + addFunctionErrors(errors); + + errors.add("NOT NULL constraint failed"); + errors.add("PRIMARY KEY or UNIQUE constraint violated"); + errors.add("duplicate key"); + errors.add("can't be cast because the value is out of range for the destination type"); + errors.add("Could not convert string"); + errors.add("Unimplemented type for cast"); + errors.add("field value out of range"); + errors.add("CHECK constraint failed"); + errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid + errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if + // the table has a primary key + errors.add("Could not cast value"); + errors.add("create unique index, table contains duplicate data"); + errors.add("Failed to cast"); + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY term out of range"); + } + +} diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java new file mode 100644 index 000000000..1e70bcb66 --- /dev/null +++ b/src/sqlancer/databend/DatabendOptions.java @@ -0,0 +1,151 @@ +package sqlancer.databend; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.databend.test.*; +import sqlancer.databend.DatabendOptions.DatabendOracleFactory; +import sqlancer.databend.test.DatabendQueryPartitioningAggregateTester; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +@Parameters(commandDescription = "Databend") +public class DatabendOptions implements DBMSSpecificOptions { + + @Parameter(names = "--test-collate", arity = 1) + public boolean testCollate = true; + + @Parameter(names = "--test-check", description = "Allow generating CHECK constraints in tables", arity = 1) + public boolean testCheckConstraints = true; + + @Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1) + public boolean testDefaultValues = true; + + @Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1) + public boolean testNotNullConstraints = true; + + @Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1) + public boolean testFunctions = true; + + @Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1) + public boolean testCasts = true; + + @Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions", arity = 1) + public boolean testBetween = true; + + @Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1) + public boolean testIn = true; + + @Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1) + public boolean testCase = true; + + @Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1) + public boolean testBinaryLogicals = true; + + @Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1) + public boolean testIntConstants = true; + + @Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1) + public boolean testStringConstants = true; + + @Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testDateConstants = true; + + @Parameter(names = "--test-timestamp-constants", description = "Allow generating TIMESTAMP constants", arity = 1) + public boolean testTimestampConstants = true; + + @Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1) + public boolean testFloatConstants = true; + + @Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1) + public boolean testBooleanConstants = true; + + @Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1) + public boolean testBinaryComparisons = true; + + @Parameter(names = "--test-indexes", description = "Allow explicit (i.e. CREATE INDEX) and implicit (i.e., UNIQUE and PRIMARY KEY) indexes", arity = 1) + public boolean testIndexes = true; + + @Parameter(names = "--test-rowid", description = "Test tables' rowid columns", arity = 1) + public boolean testRowid = true; + + @Parameter(names = "--max-num-views", description = "The maximum number of views that can be generated for a database", arity = 1) + public int maxNumViews = 1; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates = 5; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(DatabendOracleFactory.QUERY_PARTITIONING); + + public enum DatabendOracleFactory implements OracleFactory { + NOREC { + + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendNoRECOracle(globalState); + } + + }, + HAVING { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + return new DatabendQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + List oracles = new ArrayList<>(); + oracles.add(new DatabendQueryPartitioningWhereTester(globalState)); + oracles.add(new DatabendQueryPartitioningHavingTester(globalState)); + oracles.add(new DatabendQueryPartitioningAggregateTester(globalState)); + oracles.add(new DatabendQueryPartitioningDistinctTester(globalState)); + oracles.add(new DatabendQueryPartitioningGroupByTester(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + + } + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java new file mode 100644 index 000000000..fd1f2d88f --- /dev/null +++ b/src/sqlancer/databend/DatabendProvider.java @@ -0,0 +1,154 @@ +package sqlancer.databend; + +import com.google.auto.service.AutoService; +import sqlancer.*; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.databend.gen.*; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.mysql.MySQLOptions; + +import java.io.File; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +@AutoService(DatabaseProvider.class) +public class DatabendProvider extends SQLProviderAdapter { + + public DatabendProvider() { + super(DatabendGlobalState.class, DatabendOptions.class); + } + + public enum Action implements AbstractAction { + + INSERT(DatabendInsertGenerator::getQuery), // + CREATE_INDEX(DatabendIndexGenerator::getQuery), // + VACUUM((g) -> new SQLQueryAdapter("VACUUM;")), // + ANALYZE((g) -> new SQLQueryAdapter("ANALYZE;")), // + DELETE(DatabendDeleteGenerator::generate), // + UPDATE(DatabendUpdateGenerator::getQuery), // + CREATE_VIEW(DatabendViewGenerator::generate), // + EXPLAIN((g) -> { + ExpectedErrors errors = new ExpectedErrors(); + DatabendErrors.addExpressionErrors(errors); + DatabendErrors.addGroupByErrors(errors); + return new SQLQueryAdapter( + "EXPLAIN " + DatabendToStringVisitor + .asString(DatabendRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), + errors); + }); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(DatabendGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(DatabendGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case CREATE_INDEX: + if (!globalState.getDbmsSpecificOptions().testIndexes) { + return 0; + } + // fall through + case UPDATE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + case VACUUM: // seems to be ignored + case ANALYZE: // seems to be ignored + case EXPLAIN: + return r.getInteger(0, 2); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + case CREATE_VIEW: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + default: + throw new AssertionError(a); + } + } + + public static class DatabendGlobalState extends SQLGlobalState { + + @Override + protected DatabendSchema readSchema() throws SQLException { + return DatabendSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(DatabendGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new DatabendTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + DatabendProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + public void tryDeleteFile(String fname) { + try { + File f = new File(fname); + f.delete(); + } catch (Exception e) { + } + } + + public void tryDeleteDatabase(String dbpath) { + if (dbpath.equals("") || dbpath.equals(":memory:")) { + return; + } + tryDeleteFile(dbpath); + tryDeleteFile(dbpath + ".wal"); + } + + @Override + public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = MySQLOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = MySQLOptions.DEFAULT_PORT; + } + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), + globalState.getOptions().getPassword()); + Statement stmt = conn.createStatement(); + stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); + stmt.close(); + return new SQLConnection(conn); + } + + @Override + public String getDBMSName() { + return "Databend"; + } + +} diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java new file mode 100644 index 000000000..d0da77efd --- /dev/null +++ b/src/sqlancer/databend/DatabendSchema.java @@ -0,0 +1,277 @@ +package sqlancer.databend; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.*; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class DatabendSchema extends AbstractSchema { + + public enum DatabendDataType { + + INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP, NULL; + + public static DatabendDataType getRandomWithoutNull() { + DatabendDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DatabendDataType.NULL); + return dt; + } + + } + + public static class DatabendCompositeDataType { + + private final DatabendDataType dataType; + + private final int size; + + public DatabendCompositeDataType(DatabendDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public DatabendDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static DatabendCompositeDataType getRandomWithoutNull() { + DatabendDataType type = DatabendDataType.getRandomWithoutNull(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case BOOLEAN: + case VARCHAR: + case DATE: + case TIMESTAMP: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new DatabendCompositeDataType(type, size); + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 8: + return Randomly.fromOptions("BIGINT", "INT8"); + case 4: + return Randomly.fromOptions("INTEGER", "INT", "INT4", "SIGNED"); + case 2: + return Randomly.fromOptions("SMALLINT", "INT2"); + case 1: + return Randomly.fromOptions("TINYINT", "INT1"); + default: + throw new AssertionError(size); + } + case VARCHAR: + return "VARCHAR"; + case FLOAT: + switch (size) { + case 8: + return Randomly.fromOptions("DOUBLE"); + case 4: + return Randomly.fromOptions("REAL", "FLOAT4"); + default: + throw new AssertionError(size); + } + case BOOLEAN: + return Randomly.fromOptions("BOOLEAN", "BOOL"); + case TIMESTAMP: + return Randomly.fromOptions("TIMESTAMP", "DATETIME"); + case DATE: + return Randomly.fromOptions("DATE"); + case NULL: + return Randomly.fromOptions("NULL"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + } + + public static class DatabendColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + + public DatabendColumn(String name, DatabendCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class DatabendTables extends AbstractTables { + + public DatabendTables(List tables) { + super(tables); + } + + } + + public DatabendSchema(List databaseTables) { + super(databaseTables); + } + + public DatabendTables getRandomTableNonEmptyTables() { + return new DatabendTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + private static DatabendCompositeDataType getColumnType(String typeString) { + DatabendDataType primitiveType; + int size = -1; + if (typeString.startsWith("DECIMAL")) { // Ugly hack + return new DatabendCompositeDataType(DatabendDataType.FLOAT, 8); + } + switch (typeString) { + case "INTEGER": + primitiveType = DatabendDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = DatabendDataType.INT; + size = 2; + break; + case "BIGINT": + case "HUGEINT": // TODO: 16-bit int + primitiveType = DatabendDataType.INT; + size = 8; + break; + case "TINYINT": + primitiveType = DatabendDataType.INT; + size = 1; + break; + case "VARCHAR": + primitiveType = DatabendDataType.VARCHAR; + break; + case "FLOAT": + primitiveType = DatabendDataType.FLOAT; + size = 4; + break; + case "DOUBLE": + primitiveType = DatabendDataType.FLOAT; + size = 8; + break; + case "BOOLEAN": + primitiveType = DatabendDataType.BOOLEAN; + break; + case "DATE": + primitiveType = DatabendDataType.DATE; + break; + case "TIMESTAMP": + primitiveType = DatabendDataType.TIMESTAMP; + break; + case "NULL": + primitiveType = DatabendDataType.NULL; + break; + case "INTERVAL": + throw new IgnoreMeException(); + // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP + // '1969-12-29 06:50:27')) + default: + throw new AssertionError(typeString); + } + return new DatabendCompositeDataType(primitiveType, size); + } + + public static class DatabendTable extends AbstractRelationalTable { + + public DatabendTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static DatabendSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); + for (DatabendColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new DatabendSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master WHERE type='table' or type='view'")) { + while (rs.next()) { + tableNames.add(rs.getString("name")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("SELECT * FROM pragma_table_info('%s');", tableName))) { + while (rs.next()) { + String columnName = rs.getString("name"); + String dataType = rs.getString("type"); + boolean isNullable = rs.getString("notnull").contentEquals("false"); + boolean isPrimaryKey = rs.getString("pk").contains("true"); + DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); + columns.add(c); + } + } + } + if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { + // https://github.com/cwida/Databend/issues/589 + // https://github.com/cwida/Databend/issues/588 + // TODO: implement an option to enable/disable rowids + columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, false)); + } + return columns; + } + +} diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java new file mode 100644 index 000000000..50a37aeec --- /dev/null +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -0,0 +1,91 @@ +package sqlancer.databend; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; + +public class DatabendToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof DatabendConstant) { + visit((DatabendConstant) expr); + } else if (expr instanceof DatabendSelect) { + visit((DatabendSelect) expr); + } else if (expr instanceof DatabendJoin) { + visit((DatabendJoin) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(DatabendJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getOuterType() != null) { + sb.append(join.getOuterType()); + } + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(DatabendConstant constant) { + sb.append(constant.toString()); + } + + private void visit(DatabendSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(Node expr) { + DatabendToStringVisitor visitor = new DatabendToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java new file mode 100644 index 000000000..5e8d959c6 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -0,0 +1,192 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.Node; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public class DatabendConstant implements Node { + + private DatabendConstant() { + } + + public static class DatabendNullConstant extends DatabendConstant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class DatabendIntConstant extends DatabendConstant { + + private final long value; + + public DatabendIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class DatabendDoubleConstant extends DatabendConstant { + + private final double value; + + public DatabendDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + } + + public static class DatabendTextConstant extends DatabendConstant { + + private final String value; + + public DatabendTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static class DatabendBitConstant extends DatabendConstant { + + private final String value; + + public DatabendBitConstant(long value) { + this.value = Long.toBinaryString(value); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "B'" + value + "'"; + } + + } + + public static class DatabendDateConstant extends DatabendConstant { + + public String textRepr; + + public DatabendDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + } + + public static class DatabendTimestampConstant extends DatabendConstant { + + public String textRepr; + + public DatabendTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + } + + public static class DatabendBooleanConstant extends DatabendConstant { + + private final boolean value; + + public DatabendBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static Node createStringConstant(String text) { + return new DatabendTextConstant(text); + } + + public static Node createFloatConstant(double val) { + return new DatabendDoubleConstant(val); + } + + public static Node createIntConstant(long val) { + return new DatabendIntConstant(val); + } + + public static Node createNullConstant() { + return new DatabendNullConstant(); + } + + public static Node createBooleanConstant(boolean val) { + return new DatabendBooleanConstant(val); + } + + public static Node createDateConstant(long integer) { + return new DatabendDateConstant(integer); + } + + public static Node createTimestampConstant(long integer) { + return new DatabendTimestampConstant(integer); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendExpression.java b/src/sqlancer/databend/ast/DatabendExpression.java new file mode 100644 index 000000000..31b9bf859 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendExpression.java @@ -0,0 +1,5 @@ +package sqlancer.databend.ast; + +public interface DatabendExpression { + +} diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java new file mode 100644 index 000000000..ddc72ed53 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -0,0 +1,124 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.util.ArrayList; +import java.util.List; + +public class DatabendJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + private OuterType outerType; + + public enum JoinType { + INNER, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum OuterType { + FULL, LEFT, RIGHT; + + public static OuterType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public DatabendJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + private void setOuterType(OuterType outerType) { + this.outerType = outerType; + } + + public OuterType getOuterType() { + return outerType; + } + + public static List> getJoins( + List> tableList, DatabendGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); + switch (JoinType.getRandom()) { + case INNER: + joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + joinExpressions.add(DatabendJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); + break; + case LEFT: + joinExpressions + .add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions + .add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static DatabendJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DatabendJoin(left, right, JoinType.RIGHT, predicate); + } + + public static DatabendJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DatabendJoin(left, right, JoinType.LEFT, predicate); + } + + public static DatabendJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DatabendJoin(left, right, JoinType.INNER, predicate); + } + + public static Node createNaturalJoin(TableReferenceNode left, + TableReferenceNode right, OuterType naturalJoinType) { + DatabendJoin join = new DatabendJoin(left, right, JoinType.NATURAL, null); + join.setOuterType(naturalJoinType); + return join; + } + +} diff --git a/src/sqlancer/databend/ast/DatabendSelect.java b/src/sqlancer/databend/ast/DatabendSelect.java new file mode 100644 index 000000000..b3ccc08f5 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendSelect.java @@ -0,0 +1,18 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class DatabendSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + +} diff --git a/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java b/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java new file mode 100644 index 000000000..90920c819 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java @@ -0,0 +1,70 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendTable; + +public final class DatabendAlterTableGenerator { + + private DatabendAlterTableGenerator() { + } + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN + } + + public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add(" does not have a column with name \"rowid\""); + errors.add("Table does not contain column rowid referenced in alter statement"); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(DatabendCompositeDataType.getRandomWithoutNull().toString()); + break; + case ALTER_COLUMN: + sb.append("ALTER COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" SET DATA TYPE "); + sb.append(DatabendCompositeDataType.getRandomWithoutNull().toString()); + if (Randomly.getBoolean()) { + sb.append(" USING "); + DatabendErrors.addExpressionErrors(errors); + sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); + } + errors.add("Cannot change the type of this column: an index depends on it!"); + errors.add("Cannot change the type of a column that has a UNIQUE or PRIMARY KEY constraint specified"); + errors.add("Unimplemented type for cast"); + errors.add("Conversion:"); + errors.add("Cannot change the type of a column that has a CHECK constraint specified"); + break; + case DROP_COLUMN: + sb.append("DROP COLUMN "); + sb.append(table.getRandomColumn().getName()); + errors.add("named in key does not exist"); // TODO + errors.add("Cannot drop this column:"); + errors.add("Cannot drop column: table only has one column remaining!"); + errors.add("because there is a CHECK constraint that depends on it"); + errors.add("because there is a UNIQUE constraint that depends on it"); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java new file mode 100644 index 000000000..25dd83f45 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -0,0 +1,30 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendTable; + +public final class DatabendDeleteGenerator { + + private DatabendDeleteGenerator() { + } + + public static SQLQueryAdapter generate(DatabendGlobalState globalState) { + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(DatabendToStringVisitor.asString( + new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); + } + DatabendErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java new file mode 100644 index 000000000..0e7722175 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java @@ -0,0 +1,436 @@ +package sqlancer.databend.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.*; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public final class DatabendExpressionGenerator extends UntypedExpressionGenerator, DatabendColumn> { + + private final DatabendGlobalState globalState; + + public DatabendExpressionGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, CAST, FUNC, BETWEEN, CASE, + IN, COLLATE, LIKE_ESCAPE + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBoolean()) { + DatabendAggregateFunction aggregate = DatabendAggregateFunction.getRandom(); + allowAggregates = false; + return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + } + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + if (!globalState.getDbmsSpecificOptions().testCollate) { + possibleOptions.remove(Expression.COLLATE); + } + if (!globalState.getDbmsSpecificOptions().testFunctions) { + possibleOptions.remove(Expression.FUNC); + } + if (!globalState.getDbmsSpecificOptions().testCasts) { + possibleOptions.remove(Expression.CAST); + } + if (!globalState.getDbmsSpecificOptions().testBetween) { + possibleOptions.remove(Expression.BETWEEN); + } + if (!globalState.getDbmsSpecificOptions().testIn) { + possibleOptions.remove(Expression.IN); + } + if (!globalState.getDbmsSpecificOptions().testCase) { + possibleOptions.remove(Expression.CASE); + } + if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { + possibleOptions.remove(Expression.BINARY_COMPARISON); + } + if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { + possibleOptions.remove(Expression.BINARY_LOGICAL); + } + Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case COLLATE: + return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + DatabendCollate.getRandom()); + case UNARY_PREFIX: + return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), + DatabendUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + DatabendUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = DatabendBinaryComparisonOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = DatabendBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), DatabendBinaryArithmeticOperator.getRandom()); + case CAST: + return new DatabendCastOperation(generateExpression(depth + 1), + DatabendCompositeDataType.getRandomWithoutNull()); + case FUNC: + DBFunction func = DBFunction.getRandom(); + return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new NewBetweenOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new NewInOperatorNode(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new NewCaseOperatorNode(generateExpression(depth + 1), + generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), + generateExpression(depth + 1)); + case LIKE_ESCAPE: + return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), "LIKE", "ESCAPE"); + default: + throw new AssertionError(); + } + } + + @Override + protected Node generateColumn() { + DatabendColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode(column); + } + + @Override + public Node generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return DatabendConstant.createNullConstant(); + } + DatabendDataType type = DatabendDataType.getRandomWithoutNull(); + switch (type) { + case INT: + if (!globalState.getDbmsSpecificOptions().testIntConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DATE: + if (!globalState.getDbmsSpecificOptions().testDateConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createDateConstant(globalState.getRandomly().getInteger()); + case TIMESTAMP: + if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case VARCHAR: + if (!globalState.getDbmsSpecificOptions().testStringConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createStringConstant(globalState.getRandomly().getString()); + case BOOLEAN: + if (!globalState.getDbmsSpecificOptions().testBooleanConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + if (!globalState.getDbmsSpecificOptions().testFloatConstants) { + throw new IgnoreMeException(); + } + return DatabendConstant.createFloatConstant(globalState.getRandomly().getDouble()); + default: + throw new AssertionError(); + } + } + + @Override + public List> generateOrderBys() { + List> expr = super.generateOrderBys(); + List> newExpr = new ArrayList<>(expr.size()); + for (Node curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + }; + + public static class DatabendCastOperation extends NewUnaryPostfixOperatorNode { + + public DatabendCastOperation(Node expr, DatabendCompositeDataType type) { + super(expr, new Operator() { + + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + } + + } + + public enum DatabendAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1), STRING_AGG(1), FIRST(1), SUM(1), STDDEV_SAMP(1), STDDEV_POP(1), VAR_POP(1), + VAR_SAMP(1), COVAR_POP(1), COVAR_SAMP(1); + + private int nrArgs; + + DatabendAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static DatabendAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } + + public enum DBFunction { + // trigonometric functions + ACOS(1), // + ASIN(1), // + ATAN(1), // + COS(1), // + SIN(1), // + TAN(1), // + COT(1), // + ATAN2(1), // + // math functions + ABS(1), // + CEIL(1), // + CEILING(1), // + FLOOR(1), // + LOG(1), // + LOG10(1), LOG2(1), // + LN(1), // + PI(0), // + SQRT(1), // + POWER(1), // + CBRT(1), // + ROUND(2), // + SIGN(1), // + DEGREES(1), // + RADIANS(1), // + MOD(2), // + XOR(2), // + // string functions + LENGTH(1), // + LOWER(1), // + UPPER(1), // + SUBSTRING(3), // + REVERSE(1), // + CONCAT(1, true), // + CONCAT_WS(1, true), CONTAINS(2), // + PREFIX(2), // + SUFFIX(2), // + INSTR(2), // + PRINTF(1, true), // + REGEXP_MATCHES(2), // + REGEXP_REPLACE(3), // + STRIP_ACCENTS(1), // + + // date functions + DATE_PART(2), AGE(2), + + COALESCE(3), NULLIF(2), + + // LPAD(3), + // RPAD(3), + LTRIM(1), RTRIM(1), + // LEFT(2), https://github.com/cwida/Databend/issues/633 + // REPEAT(2), + REPLACE(3), UNICODE(1), + + BIT_COUNT(1), BIT_LENGTH(1), LAST_DAY(1), MONTHNAME(1), DAYNAME(1), YEARWEEK(1), DAYOFMONTH(1), WEEKDAY(1), + WEEKOFYEAR(1), + + IFNULL(2), IF(3); + + private int nrArgs; + private boolean isVariadic; + + DBFunction(int nrArgs) { + this(nrArgs, false); + } + + DBFunction(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static DBFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + + } + + public enum DatabendUnaryPostfixOperator implements Operator { + + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + DatabendUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static DatabendUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public static final class DatabendCollate implements Operator { + + private final String textRepr; + + private DatabendCollate(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return "COLLATE " + textRepr; + } + + public static DatabendCollate getRandom() { + return new DatabendCollate(DatabendTableGenerator.getRandomCollate()); + } + + } + + public enum DatabendUnaryPrefixOperator implements Operator { + + NOT("NOT"), PLUS("+"), MINUS("-"); + + private String textRepr; + + DatabendUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static DatabendUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum DatabendBinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum DatabendBinaryArithmeticOperator implements Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); + + private String textRepr; + + DatabendBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum DatabendBinaryComparisonOperator implements Operator { + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), + REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); + + private String textRepr; + + DatabendBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public NewFunctionNode generateArgsForAggregate( + DatabendAggregateFunction aggregateFunction) { + return new NewFunctionNode( + generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + } + + public Node generateAggregate() { + DatabendAggregateFunction aggrFunc = DatabendAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, DatabendUnaryPrefixOperator.NOT); + } + + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, DatabendUnaryPostfixOperator.IS_NULL); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendIndexGenerator.java b/src/sqlancer/databend/gen/DatabendIndexGenerator.java new file mode 100644 index 000000000..de6820bcc --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendIndexGenerator.java @@ -0,0 +1,59 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.util.List; + +public final class DatabendIndexGenerator { + + private DatabendIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + if (Randomly.getBoolean()) { + errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + sb.append("UNIQUE "); + } + sb.append("INDEX "); + sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information + sb.append(" ON "); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append("("); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + } + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + Node expr = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(); + sb.append(DatabendToStringVisitor.asString(expr)); + } + errors.add("already exists!"); + if (globalState.getDbmsSpecificOptions().testRowid) { + errors.add("Cannot create an index on the rowid!"); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java new file mode 100644 index 000000000..065b2e3a3 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendInsertGenerator extends AbstractInsertGenerator { + + private final DatabendGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public DatabendInsertGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + return new DatabendInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + DatabendErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertValue(DatabendColumn tiDBColumn) { + // TODO: select a more meaningful value + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append("DEFAULT"); + } else { + sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); + } + } + +} diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java new file mode 100644 index 000000000..fcc6dbd7e --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -0,0 +1,70 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public final class DatabendRandomQuerySynthesizer { + + private DatabendRandomQuerySynthesizer() { + } + + public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { + DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); + DatabendSelect select = new DatabendSelect(); + // TODO: distinct + // select.setDistinct(Randomly.getBoolean()); + // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); + List> columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + // if (allowAggregates && Randomly.getBoolean()) { + Node expression = gen.generateExpression(); + columns.add(expression); + // } else { + // columns.add(gen()); + // } + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DatabendJoin.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + + if (Randomly.getBoolean()) { + select.setLimitClause(DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause( + DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + return select; + } + +} diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java new file mode 100644 index 000000000..cd0c15b7f --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -0,0 +1,89 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendTableGenerator { + + public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(); + UntypedExpressionGenerator, DatabendColumn> gen = new DatabendExpressionGenerator( + globalState).setColumns(columns); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + sb.append(columns.get(i).getType()); + if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { + sb.append(" COLLATE "); + sb.append(getRandomCollate()); + } + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" UNIQUE"); + } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } + if (globalState.getDbmsSpecificOptions().testCheckConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" CHECK("); + sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); + DatabendErrors.addExpressionErrors(errors); + sb.append(")"); + } + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DatabendToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + } + } + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { + errors.add("Invalid type for index"); + List primaryKeyColumns = Randomly.nonEmptySubset(columns); + sb.append(", PRIMARY KEY("); + sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + } + sb.append(")"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + public static String getRandomCollate() { + return Randomly.fromOptions("NOCASE", "NOACCENT", "NOACCENT.NOCASE", "C", "POSIX"); + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + DatabendCompositeDataType columnType = DatabendCompositeDataType.getRandomWithoutNull(); + columns.add(new DatabendColumn(columnName, columnType, false, false)); + } + return columns; + } + +} diff --git a/src/sqlancer/databend/gen/DatabendUpdateGenerator.java b/src/sqlancer/databend/gen/DatabendUpdateGenerator.java new file mode 100644 index 000000000..e2f396c4b --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendUpdateGenerator.java @@ -0,0 +1,48 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; + +import java.util.List; + +public final class DatabendUpdateGenerator { + + private DatabendUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { + StringBuilder sb = new StringBuilder("UPDATE "); + ExpectedErrors errors = new ExpectedErrors(); + DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(" SET "); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append("="); + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + DatabendErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(DatabendToStringVisitor.asString(expr)); + } + DatabendErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendViewGenerator.java b/src/sqlancer/databend/gen/DatabendViewGenerator.java new file mode 100644 index 000000000..55a7c6366 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendViewGenerator.java @@ -0,0 +1,37 @@ +package sqlancer.databend.gen; + +import sqlancer.GlobalState; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; + +public final class DatabendViewGenerator { + + private DatabendViewGenerator() { + } + + public static SQLQueryAdapter generate(DatabendGlobalState globalState) { + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(DatabendToStringVisitor.asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + DatabendErrors.addExpressionErrors(errors); + DatabendErrors.addGroupByErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java new file mode 100644 index 000000000..21453005f --- /dev/null +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -0,0 +1,133 @@ +package sqlancer.databend.test; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.*; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendNoRECOracle extends NoRECBase implements TestOracle { + + private final DatabendSchema s; + + public DatabendNoRECOracle(DatabendGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + DatabendErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + DatabendTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); + Node randomWhereCondition = gen.generateExpression(); + List tables = randomTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DatabendJoin.getJoins(tableList, state); + int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); + int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, + randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getSecondQuery(List> tableList, Node randomWhereCondition, + List> joins) throws SQLException { + DatabendSelect select = new DatabendSelect(); + // select.setGroupByClause(groupBys); + // DatabendExpression isTrue = DatabendPostfixOperation.create(randomWhereCondition, + // PostfixOperator.IS_TRUE); + Node asText = new NewPostfixTextNode<>(new DatabendCastOperation( + new NewPostfixTextNode(randomWhereCondition, + " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), + new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + select.setFromList(tableList); + // select.setSelectType(SelectType.ALL); + select.setJoinList(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + DatabendToStringVisitor.asString(select) + ") as res"; + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, List> joins) + throws SQLException { + DatabendSelect select = new DatabendSelect(); + // select.setGroupByClause(groupBys); + // DatabendAggregate aggr = new DatabendAggregate( + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + // DatabendAggregateFunction.COUNT); + // select.setFetchColumns(Arrays.asList(aggr)); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new DatabendExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + // select.setSelectType(SelectType.ALL); + select.setJoinList(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = DatabendToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java new file mode 100644 index 000000000..e584da545 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -0,0 +1,188 @@ +package sqlancer.databend.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendAggregateFunction; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendBinaryArithmeticOperator; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPrefixOperator; +import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPostfixOperator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public DatabendQueryPartitioningAggregateTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + DatabendAggregateFunction aggregateFunction = Randomly.fromOptions(DatabendAggregateFunction.MAX, + DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT, + DatabendAggregateFunction.AVG/* , DatabendAggregateFunction.STDDEV_POP */); + NewFunctionNode aggregate = gen + .generateArgsForAggregate(aggregateFunction); + List> fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + originalQuery = DatabendToStringVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResult == null && secondResult != null + || firstResult != null && (!firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { + if (secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + throw new AssertionError(); + } + + } + + private String createMetamorphicUnionQuery(DatabendSelect select, + NewFunctionNode aggregate, List> from) { + String metamorphicQuery; + Node whereClause = gen.generateExpression(); + Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + DatabendUnaryPrefixOperator.NOT); + Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + DatabendUnaryPostfixOperator.IS_NULL); + List> mappedAggregate = mapped(aggregate); + DatabendSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + DatabendSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + DatabendSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += DatabendToStringVisitor.asString(leftSelect) + " UNION ALL " + + DatabendToStringVisitor.asString(middleSelect) + " UNION ALL " + + DatabendToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List> mapped(NewFunctionNode aggregate) { + DatabendCastOperation count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(Arrays.asList(aggregate)); + case AVG: + NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + DatabendAggregateFunction.SUM); + count = new DatabendCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), + new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); + return aliasArgs(Arrays.asList(sum, count)); + case STDDEV_POP: + NewFunctionNode sumSquared = new NewFunctionNode<>( + Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DatabendBinaryArithmeticOperator.MULT)), + DatabendAggregateFunction.SUM); + count = new DatabendCastOperation( + new NewFunctionNode(aggregate.getArgs(), + DatabendAggregateFunction.COUNT), + new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); + NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), + DatabendAggregateFunction.AVG); + return aliasArgs(Arrays.asList(sumSquared, count, avg)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List> aliasArgs(List> originalAggregateArgs) { + List> args = new ArrayList<>(); + int i = 0; + for (Node expr : originalAggregateArgs) { + args.add(new NewAliasNode(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(NewFunctionNode aggregate) { + switch (aggregate.getFunc()) { + case STDDEV_POP: + return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; + case AVG: + return "SUM(agg0::FLOAT)/SUM(agg1)::FLOAT"; + case COUNT: + return DatabendAggregateFunction.SUM.toString() + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private DatabendSelect getSelect(List> aggregates, List> from, + Node whereClause, List> joinList) { + DatabendSelect leftSelect = new DatabendSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java new file mode 100644 index 000000000..a806dd2ca --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -0,0 +1,82 @@ +package sqlancer.databend.test; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class DatabendQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { + + DatabendSchema s; + DatabendTables targetTables; + DatabendExpressionGenerator gen; + DatabendSelect select; + + public DatabendQueryPartitioningBase(DatabendGlobalState state) { + super(state); + DatabendErrors.addExpressionErrors(errors); + } + + public static String canonicalizeResultValue(String value) { + // Rule: -0.0 should be canonicalized to 0.0 + if (Objects.equals(value, "-0.0")) { + return "0.0"; + } + + return value; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new DatabendExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new DatabendSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DatabendJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..23572b110 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java @@ -0,0 +1,44 @@ +package sqlancer.databend.test; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class DatabendQueryPartitioningDistinctTester extends DatabendQueryPartitioningBase { + + public DatabendQueryPartitioningDistinctTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + if (Randomly.getBoolean()) { + select.setDistinct(false); + } + select.setWhereClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DatabendQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..0dbdda637 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java @@ -0,0 +1,53 @@ +package sqlancer.databend.test; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendQueryPartitioningGroupByTester extends DatabendQueryPartitioningBase { + + public DatabendQueryPartitioningGroupByTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setGroupByExpressions(select.getFetchColumns()); + select.setWhereClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DatabendQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + List> generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java new file mode 100644 index 000000000..019eeace7 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -0,0 +1,62 @@ +package sqlancer.databend.test; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase implements TestOracle { + + public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setHavingClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DatabendQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + protected Node generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List> generateFetchColumns() { + return Arrays.asList(gen.generateHavingClause()); + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java new file mode 100644 index 000000000..74f9a22c1 --- /dev/null +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java @@ -0,0 +1,45 @@ +package sqlancer.databend.test; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +public class DatabendQueryPartitioningWhereTester extends DatabendQueryPartitioningBase { + + public DatabendQueryPartitioningWhereTester(DatabendGlobalState state) { + super(state); + DatabendErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = DatabendToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setWhereClause(predicate); + String firstQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = DatabendToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = DatabendToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DatabendQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java new file mode 100644 index 000000000..a88ad740b --- /dev/null +++ b/test/sqlancer/dbms/TestDatabend.java @@ -0,0 +1,18 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestDatabend { + + @Test + public void testDatabend() { + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "databend", "--oracle", + "NoREC" })); + } + +} From e09edfaaccdba5ad446040f9197084653a1cbb45 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 28 Jun 2022 09:12:36 +0000 Subject: [PATCH 142/856] Bump checkstyle from 10.3 to 10.3.1 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3 to 10.3.1. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3...checkstyle-10.3.1) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e09cc0954..2f69dc63f 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.3 + 10.3.1 From d4c43ea5aadd7b9d6d846915d1f7a1cd61de04c0 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 30 Jun 2022 15:32:22 +0200 Subject: [PATCH 143/856] Fix ClickHouse connection string --- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 ++ src/sqlancer/clickhouse/ClickHouseProvider.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 08df43612..17993c7e4 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -35,6 +35,8 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 errors.add("Cannot parse number with a sign character but without any numeric character"); errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); + errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); + errors.add("Positional argument out of bounds"); } public static void addExpressionHavingErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 3585aa9df..7ff2de11b 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -118,7 +118,7 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ ClickHouseOptions clickHouseOptions = globalState.getDbmsSpecificOptions(); globalState.setClickHouseOptions(clickHouseOptions); - String url = String.format("jdbc:clickhouse://%s:%d/default", host, port); + String url = String.format("jdbc:clickhouse://%s:%d/%s", host, port, "default"); String databaseName = globalState.getDatabaseName(); Connection con = DriverManager.getConnection(url, globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); @@ -139,7 +139,7 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ e.printStackTrace(); } con.close(); - con = DriverManager.getConnection("jdbc:clickhouse://localhost:8123/" + databaseName, + con = DriverManager.getConnection(String.format("jdbc:clickhouse://%s:%d/%s", host, port, databaseName), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); return new SQLConnection(con); } From 87fb11d5e7ae7b97e3586edf86a92916a8b640d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Jul 2022 08:24:38 +0000 Subject: [PATCH 144/856] Bump mariadb-java-client from 3.0.5 to 3.0.6 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.5 to 3.0.6. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.5...3.0.6) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2f69dc63f..71dc5e90e 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.5 + 3.0.6 org.duckdb From 54bdd811d82fa560658f51bc961fa8e812d82065 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 1 Jul 2022 21:17:14 +0800 Subject: [PATCH 145/856] feat: implement databend connection --- .settings/org.eclipse.jdt.core.prefs | 9 ++++-- src/sqlancer/Main.java | 6 ++++ src/sqlancer/databend/DatabendOptions.java | 2 ++ src/sqlancer/databend/DatabendProvider.java | 30 ++++++++++++------- .../databend/TestDatabendConnection.java | 19 ++++++++++++ test/sqlancer/dbms/TestDatabend.java | 5 ++-- 6 files changed, 56 insertions(+), 15 deletions(-) create mode 100644 test/sqlancer/databend/TestDatabendConnection.java diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 0a659c1a7..3af1c5ad8 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.compliance=11 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,6 +28,7 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -73,6 +74,7 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -110,5 +112,6 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=1.8 +org.eclipse.jdt.core.compiler.source=11 diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a4404a6fa..6fc4aeacf 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -440,6 +440,12 @@ private String formatInteger(long intValue) { try { executorFactory.getDBMSExecutor(options.getDatabasePrefix() + "connectiontest", new Randomly()) .testConnection(); + //临时测试databend connection TODO + if(executorFactory.provider.getDBMSName() == "databend"){ + System.out.println("Databend连接成功！"); + return 0; //正常退出 + } + //----------------- } catch (Exception e) { System.err.println( "SQLancer failed creating a test database, indicating that SQLancer might have failed connecting to the DBMS. In order to change the username, password, host and port, you can use the --username, --password, --host and --port options.\n\n"); diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index 1e70bcb66..de6940a76 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -17,6 +17,8 @@ @Parameters(commandDescription = "Databend") public class DatabendOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 3307; @Parameter(names = "--test-collate", arity = 1) public boolean testCollate = true; diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index fd1f2d88f..faf136a3c 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -7,7 +7,6 @@ import sqlancer.common.query.SQLQueryProvider; import sqlancer.databend.gen.*; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.mysql.MySQLOptions; import java.io.File; import java.sql.Connection; @@ -131,24 +130,35 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE String host = globalState.getOptions().getHost(); int port = globalState.getOptions().getPort(); if (host == null) { - host = MySQLOptions.DEFAULT_HOST; + host = DatabendOptions.DEFAULT_HOST; } if (port == MainOptions.NO_SET_PORT) { - port = MySQLOptions.DEFAULT_PORT; + port = DatabendOptions.DEFAULT_PORT; } + String databaseName = globalState.getDatabaseName(); + //记录日志和执行语句处放一起或许比较合适？ TODO + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + //-------------------------------------------- String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", host, port); - Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); - Statement stmt = conn.createStatement(); - stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); - stmt.close(); - return new SQLConnection(conn); + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); } @Override public String getDBMSName() { - return "Databend"; + return "databend"; } } diff --git a/test/sqlancer/databend/TestDatabendConnection.java b/test/sqlancer/databend/TestDatabendConnection.java new file mode 100644 index 000000000..d05b33a2b --- /dev/null +++ b/test/sqlancer/databend/TestDatabendConnection.java @@ -0,0 +1,19 @@ +package sqlancer.databend; + +import com.mongodb.annotations.ThreadSafe; +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestDatabendConnection { + @Test + void testConnection(){ + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", + "--host","192.168.81.133","--port","3307","--username","user1" + ,"databend","--oracle","NoREC"})); + } +} diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index a88ad740b..269093349 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -11,8 +11,9 @@ public class TestDatabend { public void testDatabend() { assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "databend", "--oracle", - "NoREC" })); + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", + "--host","192.168.81.133","--port","3307","--username","user1" + ,"databend","--oracle","NoREC"})); } } From d4594c0876d015f2dadc143be5f0564beb842e0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Jul 2022 09:25:56 +0000 Subject: [PATCH 146/856] Bump spotbugs-maven-plugin from 4.7.0.0 to 4.7.1.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.0.0 to 4.7.1.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.0.0...spotbugs-maven-plugin-4.7.1.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 71dc5e90e..423a3ae94 100644 --- a/pom.xml +++ b/pom.xml @@ -250,7 +250,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.0.0 + 4.7.1.0 spotbugs From 23a960e9955846997cad3870eb2c182a8ac60ec1 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 22 Jul 2022 17:52:53 +0800 Subject: [PATCH 147/856] fix: fix readSchema Exception and error insert value --- src/sqlancer/GlobalState.java | 9 ++- src/sqlancer/Main.java | 6 -- src/sqlancer/databend/DatabendProvider.java | 34 +++------ src/sqlancer/databend/DatabendSchema.java | 76 ++++++++++++------- .../databend/ast/DatabendConstant.java | 8 +- .../gen/DatabendExpressionGenerator.java | 4 +- .../databend/gen/DatabendInsertGenerator.java | 16 ++-- .../databend/TestDatabendConnection.java | 3 +- test/sqlancer/dbms/TestDatabend.java | 19 ----- 9 files changed, 91 insertions(+), 84 deletions(-) delete mode 100644 test/sqlancer/dbms/TestDatabend.java diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 64b5c731e..20efcdcc8 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -5,6 +5,8 @@ import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; +import java.sql.SQLException; + public abstract class GlobalState, S extends AbstractSchema, C extends SQLancerDBConnection> { protected C databaseConnection; @@ -142,7 +144,12 @@ protected void setSchema(S schema) { } public void updateSchema() throws Exception { - setSchema(readSchema()); + try{ + setSchema(readSchema()); + } catch (SQLException sqlException) { + System.out.println(String.format("%s readSchema SQLException",databaseName)); + sqlException.printStackTrace(); + } for (AbstractTable table : schema.getDatabaseTables()) { table.recomputeCount(); } diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 6fc4aeacf..a4404a6fa 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -440,12 +440,6 @@ private String formatInteger(long intValue) { try { executorFactory.getDBMSExecutor(options.getDatabasePrefix() + "connectiontest", new Randomly()) .testConnection(); - //临时测试databend connection TODO - if(executorFactory.provider.getDBMSName() == "databend"){ - System.out.println("Databend连接成功！"); - return 0; //正常退出 - } - //----------------- } catch (Exception e) { System.err.println( "SQLancer failed creating a test database, indicating that SQLancer might have failed connecting to the DBMS. In order to change the username, password, host and port, you can use the --username, --password, --host and --port options.\n\n"); diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index faf136a3c..ffb23124a 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -24,11 +24,9 @@ public DatabendProvider() { public enum Action implements AbstractAction { INSERT(DatabendInsertGenerator::getQuery), // - CREATE_INDEX(DatabendIndexGenerator::getQuery), // - VACUUM((g) -> new SQLQueryAdapter("VACUUM;")), // - ANALYZE((g) -> new SQLQueryAdapter("ANALYZE;")), // - DELETE(DatabendDeleteGenerator::generate), // - UPDATE(DatabendUpdateGenerator::getQuery), // + //TODO 等待databend实现update && delete +// DELETE(DatabendDeleteGenerator::generate), // +// UPDATE(DatabendUpdateGenerator::getQuery), // CREATE_VIEW(DatabendViewGenerator::generate), // EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); @@ -57,19 +55,13 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { switch (a) { case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case CREATE_INDEX: - if (!globalState.getDbmsSpecificOptions().testIndexes) { - return 0; - } - // fall through - case UPDATE: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); - case VACUUM: // seems to be ignored - case ANALYZE: // seems to be ignored case EXPLAIN: return r.getInteger(0, 2); - case DELETE: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + //TODO 等待databend实现update && delete +// case UPDATE: +// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); +// case DELETE: +// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); case CREATE_VIEW: return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: @@ -136,29 +128,27 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE port = DatabendOptions.DEFAULT_PORT; } String databaseName = globalState.getDatabaseName(); - //记录日志和执行语句处放一起或许比较合适？ TODO - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - globalState.getState().logStatement("CREATE DATABASE " + databaseName); - globalState.getState().logStatement("USE " + databaseName); - //-------------------------------------------- String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", host, port); Connection con = DriverManager.getConnection(url, username, password); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); } try (Statement s = con.createStatement()) { s.execute("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); } try (Statement s = con.createStatement()) { s.execute("USE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); } return new SQLConnection(con); } @Override public String getDBMSName() { - return "databend"; + return "databend"; //用于DatabendOptions } } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index d0da77efd..5cc09914b 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -29,6 +29,14 @@ public static DatabendDataType getRandomWithoutNull() { return dt; } + public static DatabendDataType getRandomWithoutNullAndVarchar() { + DatabendDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DatabendDataType.NULL || dt == DatabendDataType.VARCHAR); + return dt; + } + } public static class DatabendCompositeDataType { @@ -82,24 +90,24 @@ public String toString() { case INT: switch (size) { case 8: - return Randomly.fromOptions("BIGINT", "INT8"); + return Randomly.fromOptions("BIGINT", "INT64"); case 4: - return Randomly.fromOptions("INTEGER", "INT", "INT4", "SIGNED"); + return Randomly.fromOptions("INT", "INT32"); case 2: - return Randomly.fromOptions("SMALLINT", "INT2"); + return Randomly.fromOptions("SMALLINT", "INT16"); case 1: - return Randomly.fromOptions("TINYINT", "INT1"); + return Randomly.fromOptions("TINYINT", "INT8"); default: throw new AssertionError(size); } case VARCHAR: - return "VARCHAR"; + return Randomly.fromOptions("VARCHAR"); case FLOAT: switch (size) { case 8: return Randomly.fromOptions("DOUBLE"); case 4: - return Randomly.fromOptions("REAL", "FLOAT4"); + return Randomly.fromOptions("FLOAT"); default: throw new AssertionError(size); } @@ -162,7 +170,7 @@ private static DatabendCompositeDataType getColumnType(String typeString) { return new DatabendCompositeDataType(DatabendDataType.FLOAT, 8); } switch (typeString) { - case "INTEGER": + case "INT": primitiveType = DatabendDataType.INT; size = 4; break; @@ -171,7 +179,6 @@ private static DatabendCompositeDataType getColumnType(String typeString) { size = 2; break; case "BIGINT": - case "HUGEINT": // TODO: 16-bit int primitiveType = DatabendDataType.INT; size = 8; break; @@ -222,12 +229,12 @@ public DatabendTable(String tableName, List columns, boolean isV public static DatabendSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); - List tableNames = getTableNames(con); + List tableNames = getTableNames(con,databaseName); for (String tableName : tableNames) { if (DBMSCommon.matchesIndexName(tableName)) { continue; // TODO: unexpected? } - List databaseColumns = getTableColumns(con, tableName); + List databaseColumns = getTableColumns(con, tableName,databaseName); boolean isView = tableName.startsWith("v"); DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); for (DatabendColumn c : databaseColumns) { @@ -239,37 +246,52 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa return new DatabendSchema(databaseTables); } - private static List getTableNames(SQLConnection con) throws SQLException { - List tableNames = new ArrayList<>(); + private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { + List tableNames = null; + tableNames = new ArrayList<>(); + //SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema != 'system' and table_schema != 'INFORMATION_SCHEMA' and table_type='BASE TABLE' + //"SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName + final String sqlStatement = String.format( + "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery("SELECT * FROM sqlite_master WHERE type='table' or type='view'")) { - while (rs.next()) { - tableNames.add(rs.getString("name")); + try (ResultSet rs = s.executeQuery(sqlStatement)) { + try{ //没有catch的话rs.next()会报SQLException：Not a navigable ResultSet + while (rs.next()) { + tableNames.add(rs.getString("table_name")); + } + } catch (Exception e){ +// e.printStackTrace(); + System.out.println("TableNames->SQLException：Not a navigable ResultSet"); } } } return tableNames; } - private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery(String.format("SELECT * FROM pragma_table_info('%s');", tableName))) { - while (rs.next()) { - String columnName = rs.getString("name"); - String dataType = rs.getString("type"); - boolean isNullable = rs.getString("notnull").contentEquals("false"); - boolean isPrimaryKey = rs.getString("pk").contains("true"); - DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); - columns.add(c); + try (ResultSet rs = s.executeQuery(String.format( + "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = '%s' and table_name ='%s'", + databaseName,tableName))) { + try{ //没有catch的话rs.next()会报SQLException：Not a navigable ResultSet + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + boolean isNullable = rs.getBoolean("is_nullable"); +// boolean isPrimaryKey = rs.getString("pk").contains("true"); + boolean isPrimaryKey = false; //没找到主键元数据 + DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); + columns.add(c); + } + } catch (Exception e) { + System.out.println("TableColumns->SQLException：Not a navigable ResultSet"); } } } if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { - // https://github.com/cwida/Databend/issues/589 - // https://github.com/cwida/Databend/issues/588 // TODO: implement an option to enable/disable rowids - columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, false)); +// columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, false)); } return columns; } diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 5e8d959c6..57ab52dba 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -57,7 +57,13 @@ public String toString() { } else if (value == Double.NEGATIVE_INFINITY) { return "'-Inf'"; } - return String.valueOf(value); + + +// DecimalFormat doubleFormat = new DecimalFormat(); +// doubleFormat.setMaximumFractionDigits(2); //小数点后2位 +// System.out.println(String.valueOf(doubleFormat.format(value))); + //TODO value先返回0.3来取消指数形式表达，等待Databend支持指数形式表达 + return String.valueOf(0.3); } } diff --git a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java index 0e7722175..1b7f5a36c 100644 --- a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java @@ -123,7 +123,9 @@ public Node generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return DatabendConstant.createNullConstant(); } - DatabendDataType type = DatabendDataType.getRandomWithoutNull(); +// DatabendDataType type = DatabendDataType.getRandomWithoutNull(); + //TODO 先跳过varchar等待databend对其更好的支持，或自己改写case VARCHAR的代码 + DatabendDataType type = DatabendDataType.getRandomWithoutNullAndVarchar(); switch (type) { case INT: if (!globalState.getDbmsSpecificOptions().testIntConstants) { diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 065b2e3a3..a61e56263 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -42,12 +42,16 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(DatabendColumn tiDBColumn) { - // TODO: select a more meaningful value - if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append("DEFAULT"); - } else { - sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); - } + // TODO: 等Databend实现NULL 和 DEFAULT ,暂时注入普通的value +// if (Randomly.getBooleanWithRatherLowProbability()) { +// sb.append("DEFAULT"); +// } else { +// sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); +// } + + String value = DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant()); + sb.append(value); + } } diff --git a/test/sqlancer/databend/TestDatabendConnection.java b/test/sqlancer/databend/TestDatabendConnection.java index d05b33a2b..91e0a1d53 100644 --- a/test/sqlancer/databend/TestDatabendConnection.java +++ b/test/sqlancer/databend/TestDatabendConnection.java @@ -3,6 +3,7 @@ import com.mongodb.annotations.ThreadSafe; import org.junit.jupiter.api.Test; import sqlancer.Main; +import sqlancer.Randomly; import sqlancer.dbms.TestConfig; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -13,7 +14,7 @@ void testConnection(){ assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", - "--host","192.168.81.133","--port","3307","--username","user1" + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC),"--host","192.168.81.134","--port","3307","--username","user1" ,"databend","--oracle","NoREC"})); } } diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java deleted file mode 100644 index 269093349..000000000 --- a/test/sqlancer/dbms/TestDatabend.java +++ /dev/null @@ -1,19 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class TestDatabend { - - @Test - public void testDatabend() { - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", - "--host","192.168.81.133","--port","3307","--username","user1" - ,"databend","--oracle","NoREC"})); - } - -} From f356d11e7a9a2ff6e40b4dbd2f185e1214f9f7ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 25 Jul 2022 09:15:47 +0000 Subject: [PATCH 148/856] Bump spotbugs-maven-plugin from 4.7.1.0 to 4.7.1.1 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.1.0 to 4.7.1.1. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.1.0...spotbugs-maven-plugin-4.7.1.1) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 423a3ae94..c86498379 100644 --- a/pom.xml +++ b/pom.xml @@ -250,7 +250,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.1.0 + 4.7.1.1 spotbugs From a34d67a26f6e79f6906e1f1c9906617d32eb459a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 07:07:38 +0000 Subject: [PATCH 149/856] Bump formatter-maven-plugin from 2.19.0 to 2.20.0 Bumps [formatter-maven-plugin](https://github.com/revelc/formatter-maven-plugin) from 2.19.0 to 2.20.0. - [Release notes](https://github.com/revelc/formatter-maven-plugin/releases) - [Changelog](https://github.com/revelc/formatter-maven-plugin/blob/main/CHANGELOG.md) - [Commits](https://github.com/revelc/formatter-maven-plugin/compare/formatter-maven-plugin-2.19.0...formatter-maven-plugin-2.20.0) --- updated-dependencies: - dependency-name: net.revelc.code.formatter:formatter-maven-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c86498379..f4221c827 100644 --- a/pom.xml +++ b/pom.xml @@ -139,7 +139,7 @@ net.revelc.code.formatter formatter-maven-plugin - 2.19.0 + 2.20.0 eclipseformat From c0332ca5a9880d8ebc4cf8e9dc754e4755da3e1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 09:24:46 +0000 Subject: [PATCH 150/856] Bump mysql-connector-java from 8.0.29 to 8.0.30 Bumps [mysql-connector-java](https://github.com/mysql/mysql-connector-j) from 8.0.29 to 8.0.30. - [Release notes](https://github.com/mysql/mysql-connector-j/releases) - [Changelog](https://github.com/mysql/mysql-connector-j/blob/release/8.0/CHANGES) - [Commits](https://github.com/mysql/mysql-connector-j/compare/8.0.29...8.0.30) --- updated-dependencies: - dependency-name: mysql:mysql-connector-java dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f4221c827..28051f42b 100644 --- a/pom.xml +++ b/pom.xml @@ -292,7 +292,7 @@ mysql mysql-connector-java - 8.0.29 + 8.0.30 org.mariadb.jdbc From c18c6f0166a4d17f0565dd38d39adf4c754594ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 14:12:28 +0000 Subject: [PATCH 151/856] Bump plexus-compiler-api from 2.12.0 to 2.12.1 Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.12.0 to 2.12.1. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.12.0...plexus-compiler-2.12.1) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28051f42b..999821ab6 100644 --- a/pom.xml +++ b/pom.xml @@ -132,7 +132,7 @@ org.codehaus.plexus plexus-compiler-api - 2.12.0 + 2.12.1 From 879daa6614cc58de6eb2dc9f5212eb71dc890fff Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Jul 2022 02:51:35 +0000 Subject: [PATCH 152/856] Bump plexus-compiler-eclipse from 2.12.0 to 2.12.1 Bumps plexus-compiler-eclipse from 2.12.0 to 2.12.1. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 999821ab6..6184911c5 100644 --- a/pom.xml +++ b/pom.xml @@ -122,7 +122,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.12.0 + 2.12.1 org.eclipse.jdt From d5d73cab478b59c5aef7cb49db077c3b4eafd60d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Jul 2022 09:15:57 +0000 Subject: [PATCH 153/856] Bump junit-jupiter-engine from 5.8.2 to 5.9.0 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.8.2 to 5.9.0. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.8.2...r5.9.0) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 6184911c5..e51f86f9a 100644 --- a/pom.xml +++ b/pom.xml @@ -307,7 +307,7 @@ org.junit.jupiter junit-jupiter-engine - 5.8.2 + 5.9.0 test From 6a5dd94f3346c227d93c6a37ea589c5d10f7ce86 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 1 Aug 2022 10:01:16 +0800 Subject: [PATCH 154/856] disable EXTRA_FLOAT_DIGITS --- .../cockroachdb/gen/CockroachDBSetSessionGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index 8aa26e88c..48bb42fb7 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -27,7 +27,7 @@ private enum CockroachDBSetting { ENABLE_ZIGZAG_JOIN(CockroachDBSetSessionGenerator::onOff), // EXPERIMENTAL_ENABLE_HASH_SHARDED_INDEXES(CockroachDBSetSessionGenerator::onOff), SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), - EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), + // EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), REORDER_JOINS_LIMIT((g) -> g.getRandomly().getInteger(0, Integer.MAX_VALUE)), // SQL_SAFE_UPDATES((g) -> "off"), // TRACING(CockroachDBSetSessionGenerator::onOff) From a0fb2187430e739f9230372c4795dd763ca24fc0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 09:20:50 +0000 Subject: [PATCH 155/856] Bump checkstyle from 10.3.1 to 10.3.2 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3.1 to 10.3.2. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3.1...checkstyle-10.3.2) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e51f86f9a..721236136 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.3.1 + 10.3.2 From d074cb9da966108f7031418e670ec51cb0775b9e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 4 Aug 2022 09:14:18 +0000 Subject: [PATCH 156/856] Bump postgresql from 42.4.0 to 42.4.1 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.4.0 to 42.4.1. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.4.0...REL42.4.1) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 721236136..9c39ee442 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.4.0 + 42.4.1 org.xerial From 9347a09dfffd54f21f3906d866a3aa2249956989 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 5 Aug 2022 09:11:53 +0000 Subject: [PATCH 157/856] Bump mariadb-java-client from 3.0.6 to 3.0.7 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.6 to 3.0.7. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.6...3.0.7) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9c39ee442..f87671fd8 100644 --- a/pom.xml +++ b/pom.xml @@ -297,7 +297,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.6 + 3.0.7 org.duckdb From 9195b29605cbaccacc7de4efe92e264f191c74e7 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 12 Aug 2022 21:11:44 +0800 Subject: [PATCH 158/856] fix: comment out some expressions not yet supported by databend --- src/sqlancer/databend/DatabendProvider.java | 8 +- src/sqlancer/databend/DatabendSchema.java | 27 ++-- .../gen/DatabendExpressionGenerator.java | 133 ++++++++++-------- .../gen/DatabendRandomQuerySynthesizer.java | 2 +- .../databend/gen/DatabendTableGenerator.java | 47 ++++--- .../databend/test/DatabendNoRECOracle.java | 8 +- 6 files changed, 121 insertions(+), 104 deletions(-) diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index ffb23124a..241bb136a 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -27,7 +27,7 @@ public enum Action implements AbstractAction { //TODO 等待databend实现update && delete // DELETE(DatabendDeleteGenerator::generate), // // UPDATE(DatabendUpdateGenerator::getQuery), // - CREATE_VIEW(DatabendViewGenerator::generate), // +// CREATE_VIEW(DatabendViewGenerator::generate), // TODO 等待databend的create view语法 更加贴近mysql EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); @@ -62,8 +62,8 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); // case DELETE: // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); - case CREATE_VIEW: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); +// case CREATE_VIEW: +// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); } @@ -96,7 +96,7 @@ public void generateDatabase(DatabendGlobalState globalState) throws Exception { throw new IgnoreMeException(); } }); - se.executeStatements(); + se.executeStatements(); //在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 } public void tryDeleteFile(String fname) { diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 5cc09914b..ad92932d2 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -19,7 +19,8 @@ public class DatabendSchema extends AbstractSchema generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { +// if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { +// return generateLeafNode(); +// } + if (depth >= 1) { return generateLeafNode(); } - if (allowAggregates && Randomly.getBoolean()) { - DatabendAggregateFunction aggregate = DatabendAggregateFunction.getRandom(); - allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); - } +// if (allowAggregates && Randomly.getBoolean()) { +// DatabendAggregateFunction aggregate = DatabendAggregateFunction.getRandom(); +// allowAggregates = false; +// return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); +// } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - if (!globalState.getDbmsSpecificOptions().testCollate) { - possibleOptions.remove(Expression.COLLATE); - } - if (!globalState.getDbmsSpecificOptions().testFunctions) { - possibleOptions.remove(Expression.FUNC); - } - if (!globalState.getDbmsSpecificOptions().testCasts) { - possibleOptions.remove(Expression.CAST); - } +// if (!globalState.getDbmsSpecificOptions().testCollate) { +// possibleOptions.remove(Expression.COLLATE); +// } +// if (!globalState.getDbmsSpecificOptions().testFunctions) { +// possibleOptions.remove(Expression.FUNC); +// } +// if (!globalState.getDbmsSpecificOptions().testCasts) { +// possibleOptions.remove(Expression.CAST); +// } if (!globalState.getDbmsSpecificOptions().testBetween) { possibleOptions.remove(Expression.BETWEEN); } if (!globalState.getDbmsSpecificOptions().testIn) { possibleOptions.remove(Expression.IN); } - if (!globalState.getDbmsSpecificOptions().testCase) { - possibleOptions.remove(Expression.CASE); - } +// if (!globalState.getDbmsSpecificOptions().testCase) { +// possibleOptions.remove(Expression.CASE); +// } if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { possibleOptions.remove(Expression.BINARY_COMPARISON); } @@ -67,9 +77,9 @@ protected Node generateExpression(int depth) { } Expression expr = Randomly.fromList(possibleOptions); switch (expr) { - case COLLATE: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), - DatabendCollate.getRandom()); +// case COLLATE: +// return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), +// DatabendCollate.getRandom()); case UNARY_PREFIX: return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), DatabendUnaryPrefixOperator.getRandom()); @@ -84,30 +94,31 @@ protected Node generateExpression(int depth) { op = DatabendBinaryLogicalOperator.getRandom(); return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), op); - case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), DatabendBinaryArithmeticOperator.getRandom()); - case CAST: - return new DatabendCastOperation(generateExpression(depth + 1), - DatabendCompositeDataType.getRandomWithoutNull()); - case FUNC: - DBFunction func = DBFunction.getRandom(); - return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); +// case BINARY_ARITHMETIC: +// return new NewBinaryOperatorNode(generateExpression(depth + 1), +// generateExpression(depth + 1), DatabendBinaryArithmeticOperator.getRandom()); +// case CAST: +// return new DatabendCastOperation(generateExpression(depth + 1), +// DatabendCompositeDataType.getRandomWithoutNull()); +// case FUNC: +// DBFunction func = DBFunction.getRandom(); +// return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); case BETWEEN: return new NewBetweenOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); case IN: return new NewInOperatorNode(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); - case CASE: - int nr = Randomly.smallNumber() + 1; - return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), - generateExpression(depth + 1)); - case LIKE_ESCAPE: - return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), - generateExpression(depth + 1), "LIKE", "ESCAPE"); +// case CASE: +// int nr = Randomly.smallNumber() + 1; +// return new NewCaseOperatorNode(generateExpression(depth + 1), +// generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), +// generateExpression(depth + 1)); +// case LIKE_ESCAPE: +// return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), +// generateExpression(depth + 1), "LIKE", "ESCAPE"); default: + System.out.println(expr.toString()); throw new AssertionError(); } } @@ -132,16 +143,16 @@ public Node generateConstant() { throw new IgnoreMeException(); } return DatabendConstant.createIntConstant(globalState.getRandomly().getInteger()); - case DATE: - if (!globalState.getDbmsSpecificOptions().testDateConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createDateConstant(globalState.getRandomly().getInteger()); - case TIMESTAMP: - if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createTimestampConstant(globalState.getRandomly().getInteger()); +// case DATE: +// if (!globalState.getDbmsSpecificOptions().testDateConstants) { +// throw new IgnoreMeException(); +// } +// return DatabendConstant.createDateConstant(globalState.getRandomly().getInteger()); +// case TIMESTAMP: +// if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { +// throw new IgnoreMeException(); +// } +// return DatabendConstant.createTimestampConstant(globalState.getRandomly().getInteger()); case VARCHAR: if (!globalState.getDbmsSpecificOptions().testStringConstants) { throw new IgnoreMeException(); @@ -190,8 +201,9 @@ public String getTextRepresentation() { } public enum DatabendAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1), STRING_AGG(1), FIRST(1), SUM(1), STDDEV_SAMP(1), STDDEV_POP(1), VAR_POP(1), - VAR_SAMP(1), COVAR_POP(1), COVAR_SAMP(1); + MAX(1), MIN(1), AVG(1), COUNT(1),SUM(1), STDDEV_POP(1), + COVAR_POP(1), COVAR_SAMP(2); + //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) private int nrArgs; @@ -210,7 +222,7 @@ public int getNrArgs() { } public enum DBFunction { - // trigonometric functions + // trigonometric functions, nrArgs表示参数个数 ACOS(1), // ASIN(1), // ATAN(1), // @@ -218,7 +230,7 @@ public enum DBFunction { SIN(1), // TAN(1), // COT(1), // - ATAN2(1), // + ATAN2(2), // // math functions ABS(1), // CEIL(1), // @@ -265,8 +277,9 @@ public enum DBFunction { // REPEAT(2), REPLACE(3), UNICODE(1), - BIT_COUNT(1), BIT_LENGTH(1), LAST_DAY(1), MONTHNAME(1), DAYNAME(1), YEARWEEK(1), DAYOFMONTH(1), WEEKDAY(1), + BIT_COUNT(1), BIT_LENGTH(1), MONTHNAME(1), DAYNAME(1), YEARWEEK(1), DAYOFMONTH(1), WEEKDAY(1), WEEKOFYEAR(1), + //, LAST_DAY(1) IFNULL(2), IF(3); @@ -338,7 +351,8 @@ public static DatabendCollate getRandom() { public enum DatabendUnaryPrefixOperator implements Operator { - NOT("NOT"), PLUS("+"), MINUS("-"); + NOT("NOT"); + //PLUS("+"), MINUS("-"); //等待boolean支持数学运算 private String textRepr; @@ -373,7 +387,7 @@ public static Operator getRandom() { } public enum DatabendBinaryArithmeticOperator implements Operator { - CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); + ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"); private String textRepr; @@ -393,9 +407,8 @@ public String getTextRepresentation() { } public enum DatabendBinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), - LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), - REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); + //LIKE("LIKE"), NOT_LIKE("NOT LIKE"); private String textRepr; diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index fcc6dbd7e..eebb9ffe3 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -25,7 +25,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); - // TODO: distinct + // TODO distinct // select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); List> columns = new ArrayList<>(); diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index cd0c15b7f..0dd75375a 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -36,38 +36,41 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType()); - if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() - && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { - sb.append(" COLLATE "); - sb.append(getRandomCollate()); - } - if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" UNIQUE"); - } +// if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() +// && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { +// sb.append(" COLLATE "); +// sb.append(getRandomCollate()); +// } +// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { +// sb.append(" UNIQUE"); +// } if (globalState.getDbmsSpecificOptions().testNotNullConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); + } else { + sb.append(" NULL"); //Databend 默认字段为非空，这个将它默认设置为允许空 } - if (globalState.getDbmsSpecificOptions().testCheckConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" CHECK("); - sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); - DatabendErrors.addExpressionErrors(errors); - sb.append(")"); - } +// if (globalState.getDbmsSpecificOptions().testCheckConstraints //databend 无check约束 +// && Randomly.getBooleanWithRatherLowProbability()) { +// sb.append(" CHECK("); +// sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); +// DatabendErrors.addExpressionErrors(errors); +// sb.append(")"); +// } if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DatabendToStringVisitor.asString(gen.generateConstant())); sb.append(")"); } } - if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { - errors.add("Invalid type for index"); - List primaryKeyColumns = Randomly.nonEmptySubset(columns); - sb.append(", PRIMARY KEY("); - sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - } + //databend并没有索引 +// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { +// errors.add("Invalid type for index"); +// List primaryKeyColumns = Randomly.nonEmptySubset(columns); +// sb.append(", PRIMARY KEY("); +// sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); +// sb.append(")"); +// } sb.append(")"); return new SQLQueryAdapter(sb.toString(), errors, true); } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 21453005f..858f411a0 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -41,15 +41,15 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { @Override public void check() throws SQLException { - DatabendTables randomTables = s.getRandomTableNonEmptyTables(); + DatabendTables randomTables = s.getRandomTableNonEmptyTables(); //随机获得nr张表 List columns = randomTables.getColumns(); DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); + Node randomWhereCondition = gen.generateExpression(); //生成随机where条件，形式为ast List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); List> joins = DatabendJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); + int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); //禁用优化 int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { @@ -71,7 +71,7 @@ private int getSecondQuery(List> tableList, Node(randomWhereCondition, " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); - select.setFetchColumns(Arrays.asList(asText)); + select.setFetchColumns(Arrays.asList(asText)); // ? select.setFromList(tableList); // select.setSelectType(SelectType.ALL); select.setJoinList(joins); From abd932a2154f574af413fe6531c81bd6f3924328 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:01:19 +0800 Subject: [PATCH 159/856] feat: implement the binary comparison operation --- .../DatabendBinaryComparisonOperation.java | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java diff --git a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java new file mode 100644 index 000000000..f76579ca3 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java @@ -0,0 +1,37 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; + + +public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode { + + public DatabendBinaryComparisonOperation(Node left, Node right, + DatabendBinaryComparisonOperator op) { + super(left, right, op); + } + + public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Operator{ + EQUALS("="), +// IS_DISTINCT("IS DISTINCT FROM"), +// IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), + NOT_EQUALS("!="), + LESS("<"), + LESS_EQUALS("<="), + GREATER(">"), + GREATER_EQUALS(">="); + + DatabendBinaryComparisonOperator(String text) { + textRepresentation = text; + } + private String textRepresentation; + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + + +} From c3da03ec3722325f52179164bdc64a1e2ad89457 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:02:28 +0800 Subject: [PATCH 160/856] feat: implement the binary logical operation --- .../ast/DatabendBinaryLogicalOperation.java | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java diff --git a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java new file mode 100644 index 000000000..32c8df0ca --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java @@ -0,0 +1,76 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; + +public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode { + + public DatabendBinaryLogicalOperation(Node left, Node right, + DatabendBinaryLogicalOperator op) { + super(left,right,op); + } + + public enum DatabendBinaryLogicalOperator implements BinaryOperatorNode.Operator { + AND("AND", "and") { +// @Override +// public DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right) { +// if (left.isNull() && right.isNull()) { +// return DatabendNoRECConstant.createNullConstant(); +// } else if (left.isNull()) { +// if (right.asBooleanNotNull()) { +// return DatabendNoRECConstant.createNullConstant(); +// } else { +// return DatabendNoRECConstant.createFalse(); +// } +// } else if (right.isNull()) { +// if (left.asBooleanNotNull()) { +// return DatabendNoRECConstant.createNullConstant(); +// } else { +// return DatabendNoRECConstant.createFalse(); +// } +// } else { +// return left.asBooleanNotNull() && right.asBooleanNotNull() ? DatabendNoRECConstant.createTrue() +// : DatabendNoRECConstant.createFalse(); +// } +// } + }, + OR("OR", "or") { +// @Override +// public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { +// if (!left.isNull() && left.asBooleanNotNull()) { +// return ClickHouseConstant.createTrue(); +// } else if (!right.isNull() && right.asBooleanNotNull()) { +// return ClickHouseConstant.createTrue(); +// } else if (left.isNull() || right.isNull()) { +// return ClickHouseConstant.createNullConstant(); +// } else { +// return ClickHouseConstant.createFalse(); +// } +// } + }; + + private final String[] textRepresentations; + + DatabendBinaryLogicalOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + + public DatabendBinaryLogicalOperator getRandomOp() { + return Randomly.fromOptions(values()); + } + +// public abstract DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right); + + public static DatabendBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + +} From a717c5df34969d59d6c8d15e3011dd8103a94f31 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:03:30 +0800 Subject: [PATCH 161/856] feat: implement the like operation --- .../databend/ast/DatabendLikeOperation.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendLikeOperation.java diff --git a/src/sqlancer/databend/ast/DatabendLikeOperation.java b/src/sqlancer/databend/ast/DatabendLikeOperation.java new file mode 100644 index 000000000..e76219b1a --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendLikeOperation.java @@ -0,0 +1,31 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; + +public class DatabendLikeOperation extends NewBinaryOperatorNode { + + public DatabendLikeOperation(Node left, Node right + , DatabendLikeOperator op) { + super(left, right, op); + } + + + public enum DatabendLikeOperator implements BinaryOperatorNode.Operator { + LIKE_OPERATOR("LIKE","like"); + + private final String[] textRepresentations; + + DatabendLikeOperator(String... text){ + textRepresentations = text; + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + } + +} From 9b3aac126ac18a26fab3aaaecd8dd121728a10a0 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:04:07 +0800 Subject: [PATCH 162/856] feat: implement the unary postfix operation --- .../ast/DatabendUnaryPostfixOperation.java | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java new file mode 100644 index 000000000..e20a6be6b --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -0,0 +1,77 @@ +package sqlancer.databend.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendSchema.DatabendDataType; + + +public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { + +// private final Node expr; +// private final DatabendUnaryPostfixOperator op; + private boolean negate; + + public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op, boolean negate) { + super(expr,op); + setNegate(negate); + } + + public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL"){ + @Override + public DatabendDataType[] getInputDataTypes() { + return DatabendDataType.values(); + } + }, + IS_NOT_NULL("IS NOT NULL"){ + @Override + public DatabendDataType[] getInputDataTypes() { + return DatabendDataType.values(); + } + }; + //IS + + private final String textRepresentations; + + DatabendUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static DatabendUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract DatabendDataType[] getInputDataTypes(); + + } + + public boolean isNegated() { + return negate; + } + + public void setNegate(boolean negate) { + this.negate = negate; + } + +// @Override + public Node getExpression() { + return getExpr(); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + +// @Override +// public OperatorKind getOperatorKind() { +// return OperatorKind.POSTFIX; +// } +} From e2d52ff4a06fed7af3ea868e9675d65477d6d98f Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:05:18 +0800 Subject: [PATCH 163/856] feat: implement the unary prefix operation --- .../ast/DatabendUnaryPrefixOperation.java | 77 +++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java new file mode 100644 index 000000000..eabb4eabd --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -0,0 +1,77 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendSchema.DatabendDataType; + + +public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { + +// private final Node expr; +// private final DatabendUnaryPrefixOperator op; + private boolean negate; + + public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op, boolean negate) { + super(expr,op); + setNegate(negate); + } + + void setNegate(boolean negate){ + this.negate = negate; + } + +// @Override + public Node getExpression() { + return getExpr(); + } + + @Override + public String getOperatorRepresentation() { + return null; + } + +// @Override +// public OperatorKind getOperatorKind() { +// return OperatorKind.PREFIX; +// } + + + + public enum DatabendUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", DatabendDataType.BOOLEAN, DatabendDataType.INT) { + @Override + public DatabendDataType getExpressionType() { + return DatabendDataType.BOOLEAN; + } + + @Override + protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { + return null; // TODO + } + }; + + + private String textRepresentation; + private DatabendDataType[] dataTypes; + + DatabendUnaryPrefixOperator(String textRepresentation, DatabendDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract DatabendDataType getExpressionType(); + + public DatabendDataType[] getInputDataTypes(){ + return dataTypes; + } + + protected abstract DatabendConstant getExpectedValue(DatabendConstant expectedValue); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + } + +} From ccf79d8cd6f38e2c83cbd1166357d6d8c048d0dd Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:08:20 +0800 Subject: [PATCH 164/856] feat: refactored untypde expression generator to typed expression generator --- .../gen/DatabendNoRECExpressionGenerator.java | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java new file mode 100644 index 000000000..4c65ccf8b --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -0,0 +1,213 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.ast.*; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; +import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; +import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +public class DatabendNoRECExpressionGenerator extends + TypedExpressionGenerator, DatabendColumn, DatabendDataType> { + + private final DatabendGlobalState globalState; + + public DatabendNoRECExpressionGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; +// SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; + } + + public Node generateLeafNode(DatabendDataType dataType) { + return generateConstant(dataType); + } + + + @Override + protected Node generateExpression(DatabendDataType type, int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } + + switch (type) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + case FLOAT: + case VARCHAR: + case NULL: + return generateConstant(type); + default: + throw new AssertionError(); + } + } + + private Node generateIntExpression(int depth) { + + return null; + } + + Node generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + getPostfix(depth + 1); + case NOT: + getNOT(depth + 1); + case BETWEEN: + return getBetween(depth + 1); + case IN_OPERATION: + return getIn(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return getBinaryLogical(depth + 1,DatabendDataType.BOOLEAN); + case BINARY_COMPARISON: + return getComparison(depth + 1); + case LIKE: + return getLike(depth + 1,DatabendDataType.VARCHAR); + default: + throw new AssertionError(); + } + + } + + Node getPostfix(int depth) { + DatabendUnaryPostfixOperator randomOp = DatabendUnaryPostfixOperator.getRandom(); +// System.out.println("getPostfix:" + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth)); + return new DatabendUnaryPostfixOperation( + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), + randomOp,Randomly.getBoolean()); + } + + Node getNOT(int depth) { + DatabendUnaryPrefixOperator op = DatabendUnaryPrefixOperator.NOT; + return new DatabendUnaryPrefixOperation( + generateExpression(Randomly.fromOptions(op.getInputDataTypes()), depth), + op,Randomly.getBoolean()); + } + + Node getBetween(int depth){ + //跳过boolean + DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() + .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); + + return new NewBetweenOperatorNode(generateExpression(dataType,depth), + generateExpression(dataType,depth), generateExpression(dataType,depth), + Randomly.getBoolean()); + } + + Node getIn(int depth) { + DatabendDataType dataType = Randomly.fromOptions(DatabendDataType.values()); + Node leftExpr = generateExpression(dataType,depth); + List> rightExprs = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + for(int i = 0; i < nr; i++) { + rightExprs.add(generateExpression(dataType,depth)); + } + return new NewInOperatorNode(leftExpr,rightExprs, Randomly.getBoolean()); + } + + Node getBinaryLogical(int depth, DatabendDataType dataType){ + Node expr = generateExpression(dataType,depth); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + expr = new DatabendBinaryLogicalOperation(expr, + generateExpression(DatabendDataType.BOOLEAN, depth), + DatabendBinaryLogicalOperator.getRandom()); + } + return expr; + } + + Node getComparison(int depth) { + //跳过boolean + DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() + .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); + Node leftExpr = generateExpression(dataType,depth); + Node rightExpr = generateExpression(dataType,depth); + DatabendBinaryComparisonOperation op = new DatabendBinaryComparisonOperation(leftExpr,rightExpr, + Randomly.fromOptions(DatabendBinaryComparisonOperator.values())); + return op; + } + + Node getLike(int depth, DatabendDataType dataType) { + return new DatabendLikeOperation(generateExpression(dataType,depth) + ,generateExpression(dataType,depth), DatabendLikeOperation.DatabendLikeOperator.LIKE_OPERATOR); + } + + @Override + public Node generatePredicate() { + return null; + } + + @Override + public Node negatePredicate(Node predicate) { + return null; + } + + @Override + public Node isNull(Node expr) { + return null; + } + + @Override + public Node generateConstant(DatabendDataType type) { + Randomly r = globalState.getRandomly(); + if (Randomly.getBooleanWithSmallProbability()) { + return DatabendConstant.createNullConstant(); + } + + switch (type) { + case INT: + if (Randomly.getBooleanWithSmallProbability()) { + return DatabendConstant.createStringConstant(String.valueOf(r.getInteger())); + } else { + return DatabendConstant.createIntConstant(r.getInteger()); + } + case BOOLEAN: + if (Randomly.getBooleanWithSmallProbability()) { + return DatabendConstant + .createStringConstant(Randomly.fromOptions("TRUE", "FALSE")); //TODO databend中非0为false，0为false + } else { + return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); + } + case FLOAT: + return DatabendConstant.createFloatConstant((float) r.getDouble()); + case VARCHAR: + return DatabendConstant.createStringConstant(r.getString()); + case NULL: + return DatabendConstant.createNullConstant(); + default: + throw new AssertionError(type); + } + } + + @Override + protected Node generateColumn(DatabendDataType type) { + return null; + } + + @Override + protected DatabendDataType getRandomType() { + return Randomly.fromOptions(DatabendDataType.values()); + } + + @Override + protected boolean canGenerateColumnOfType(DatabendDataType type) { + return false; + } +} From fb86d9bd7670c10dfb3be3311ef2084aac26914c Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:12:07 +0800 Subject: [PATCH 165/856] fix: replace the expression generator --- src/sqlancer/databend/ast/DatabendJoin.java | 20 +++++++++++-------- .../gen/DatabendRandomQuerySynthesizer.java | 16 +++++++++------ .../databend/test/DatabendNoRECOracle.java | 14 +++++++++---- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index ddc72ed53..886f3824c 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -3,10 +3,11 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.gen.DatabendExpressionGenerator; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; import java.util.ArrayList; import java.util.List; @@ -28,7 +29,7 @@ public static JoinType getRandom() { } public enum OuterType { - FULL, LEFT, RIGHT; + LEFT, RIGHT; public static OuterType getRandom() { return Randomly.fromOptions(values()); @@ -76,21 +77,24 @@ public static List> getJoins( TableReferenceNode rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); - DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); +// DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); + DatabendNoRECExpressionGenerator joinGen = new DatabendNoRECExpressionGenerator(globalState).setColumns(columns); + switch (JoinType.getRandom()) { case INNER: - joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case NATURAL: joinExpressions.add(DatabendJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); break; case LEFT: - joinExpressions - .add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + joinExpressions.add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case RIGHT: - joinExpressions - .add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + joinExpressions.add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; default: throw new AssertionError(); diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index eebb9ffe3..a0b4c087b 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -3,6 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -22,7 +23,9 @@ private DatabendRandomQuerySynthesizer() { public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); - DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) +// DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) +// .setColumns(targetTables.getColumns()); + DatabendNoRECExpressionGenerator gen = new DatabendNoRECExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); // TODO distinct @@ -31,7 +34,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int List> columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = gen.generateExpression(); + Node expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); columns.add(expression); // } else { // columns.add(gen()); @@ -45,7 +48,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } if (Randomly.getBoolean()) { select.setOrderByExpressions(gen.generateOrderBys()); @@ -61,9 +64,10 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setOffsetClause( DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } - if (Randomly.getBoolean()) { - select.setHavingClause(gen.generateHavingClause()); - } + // TODO 待添加HavingClause +// if (Randomly.getBoolean()) { +// select.setHavingClause(gen.generateHavingClause()); +// } return select; } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 858f411a0..d514164ae 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -21,6 +21,7 @@ import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.*; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; +import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; import java.sql.ResultSet; import java.sql.SQLException; @@ -43,8 +44,13 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { public void check() throws SQLException { DatabendTables randomTables = s.getRandomTableNonEmptyTables(); //随机获得nr张表 List columns = randomTables.getColumns(); - DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); //生成随机where条件，形式为ast +// DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); + DatabendNoRECExpressionGenerator gen = new DatabendNoRECExpressionGenerator(state).setColumns(columns); + + Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); //生成随机where条件，形式为ast + +// System.out.println(DatabendToStringVisitor.asString(randomWhereCondition)); + List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -71,13 +77,14 @@ private int getSecondQuery(List> tableList, Node(randomWhereCondition, " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); // ? select.setFromList(tableList); - // select.setSelectType(SelectType.ALL); select.setJoinList(joins); int secondCount = 0; unoptimizedQueryString = "SELECT SUM(count) FROM (" + DatabendToStringVisitor.asString(select) + ") as res"; errors.add("canceling statement due to statement timeout"); +// System.out.println(unoptimizedQueryString); SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { @@ -111,7 +118,6 @@ private int getFirstQueryCount(SQLConnection con, List> if (Randomly.getBooleanWithSmallProbability()) { select.setOrderByExpressions(new DatabendExpressionGenerator(state).setColumns(columns).generateOrderBys()); } - // select.setSelectType(SelectType.ALL); select.setJoinList(joins); int firstCount = 0; try (Statement stat = con.createStatement()) { From 9f260f3f04dcb6a8ac5f79c7ccec8d01aee2fb51 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 12:28:46 +0800 Subject: [PATCH 166/856] fix: update double constant --- src/sqlancer/databend/ast/DatabendConstant.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 57ab52dba..7b0d09164 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -59,11 +59,7 @@ public String toString() { } -// DecimalFormat doubleFormat = new DecimalFormat(); -// doubleFormat.setMaximumFractionDigits(2); //小数点后2位 -// System.out.println(String.valueOf(doubleFormat.format(value))); - //TODO value先返回0.3来取消指数形式表达，等待Databend支持指数形式表达 - return String.valueOf(0.3); + return String.valueOf(value).toLowerCase(); // TODO 待databend支持大写E再去掉小写转换 } } From 71a27693b802860bb8c9ff8fb78c15201d039928 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 19 Aug 2022 16:22:39 +0800 Subject: [PATCH 167/856] fix: insert null on not null column --- src/sqlancer/databend/gen/DatabendInsertGenerator.java | 10 ++++++---- .../databend/gen/DatabendNoRECExpressionGenerator.java | 10 ++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index a61e56263..1e61d5a8c 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -5,6 +5,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -41,15 +42,16 @@ private SQLQueryAdapter generate() { } @Override - protected void insertValue(DatabendColumn tiDBColumn) { - // TODO: 等Databend实现NULL 和 DEFAULT ,暂时注入普通的value + protected void insertValue(DatabendColumn column) { + // TODO: 等Databend实现DEFAULT ,暂时注入普通的value // if (Randomly.getBooleanWithRatherLowProbability()) { // sb.append("DEFAULT"); // } else { // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); // } - - String value = DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant()); + String value = DatabendToStringVisitor.asString( + new DatabendNoRECExpressionGenerator(globalState). + generateConstant(column.isNullable())); sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java index 4c65ccf8b..53e2d526e 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -1,5 +1,6 @@ package sqlancer.databend.gen; +import com.google.common.collect.RangeMap; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.common.ast.newast.NewInOperatorNode; @@ -164,6 +165,15 @@ public Node isNull(Node expr) { return null; } + public Node generateConstant(boolean isNullable) { + DatabendDataType type; + do { + type = Randomly.fromOptions(DatabendDataType.values()); + } while(!isNullable && type.equals(DatabendDataType.NULL)); + + return generateConstant(type); + } + @Override public Node generateConstant(DatabendDataType type) { Randomly r = globalState.getRandomly(); From 55f8d81fab5f0633122d702189a1bea52b1eb397 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 19 Aug 2022 09:12:36 +0000 Subject: [PATCH 168/856] Bump postgresql from 42.4.1 to 42.4.2 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.4.1 to 42.4.2. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.4.1...REL42.4.2) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f87671fd8..86e589805 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.4.1 + 42.4.2 org.xerial From 81c37555941a42aeb9fa4fa3fa943721571342a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Aug 2022 09:13:43 +0000 Subject: [PATCH 169/856] Bump maven-jxr-plugin from 3.2.0 to 3.3.0 Bumps [maven-jxr-plugin](https://github.com/apache/maven-jxr) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/apache/maven-jxr/releases) - [Commits](https://github.com/apache/maven-jxr/compare/jxr-3.2.0...jxr-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jxr-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 86e589805..28a057022 100644 --- a/pom.xml +++ b/pom.xml @@ -341,7 +341,7 @@ org.apache.maven.plugins maven-jxr-plugin - 3.2.0 + 3.3.0 From d4138a9b32eea3dd7544cafb7d56d6361a6c6283 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Tue, 23 Aug 2022 09:49:25 +0800 Subject: [PATCH 170/856] fix: remove int and boolean constants of type string --- .../gen/DatabendNoRECExpressionGenerator.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java index 53e2d526e..c24a9b246 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -183,18 +183,11 @@ public Node generateConstant(DatabendDataType type) { switch (type) { case INT: - if (Randomly.getBooleanWithSmallProbability()) { - return DatabendConstant.createStringConstant(String.valueOf(r.getInteger())); - } else { - return DatabendConstant.createIntConstant(r.getInteger()); - } + //不支持string转化故直接返回int constant + return DatabendConstant.createIntConstant(r.getInteger()); case BOOLEAN: - if (Randomly.getBooleanWithSmallProbability()) { - return DatabendConstant - .createStringConstant(Randomly.fromOptions("TRUE", "FALSE")); //TODO databend中非0为false，0为false - } else { - return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); - } + //不支持string转化故直接返回boolean constant + return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); case FLOAT: return DatabendConstant.createFloatConstant((float) r.getDouble()); case VARCHAR: From 282f712c2a2948a096ff313ea62da0e61d4ed451 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Aug 2022 02:28:17 +0000 Subject: [PATCH 171/856] Bump maven-javadoc-plugin from 3.4.0 to 3.4.1 Bumps [maven-javadoc-plugin](https://github.com/apache/maven-javadoc-plugin) from 3.4.0 to 3.4.1. - [Release notes](https://github.com/apache/maven-javadoc-plugin/releases) - [Commits](https://github.com/apache/maven-javadoc-plugin/compare/maven-javadoc-plugin-3.4.0...maven-javadoc-plugin-3.4.1) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-javadoc-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 28a057022..8d4ce21f1 100644 --- a/pom.xml +++ b/pom.xml @@ -397,7 +397,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.4.0 + 3.4.1 8 From 091fc3c1f7007fba53c7fc46d34aa7faf8cd9561 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Thu, 25 Aug 2022 01:02:52 +0800 Subject: [PATCH 172/856] fix: insert appropriate constants --- src/sqlancer/databend/DatabendProvider.java | 6 ++++++ .../databend/gen/DatabendInsertGenerator.java | 4 ++-- .../gen/DatabendNoRECExpressionGenerator.java | 16 ++++++++-------- .../databend/gen/DatabendTableGenerator.java | 9 ++++++--- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 241bb136a..dd54a56db 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -143,6 +143,12 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE s.execute("USE " + databaseName); globalState.getState().logStatement("USE " + databaseName); } + +// try (Statement s = con.createStatement()) { +// s.execute("set enable_planner_v2 = 0;"); +// globalState.getState().logStatement("set enable_planner_v2 = 0;"); +// } + return new SQLConnection(con); } diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 1e61d5a8c..d05f65c14 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -43,7 +43,7 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(DatabendColumn column) { - // TODO: 等Databend实现DEFAULT ,暂时注入普通的value + // TODO: 等Databend实现DEFAULT关键字,暂时注入普通的value // if (Randomly.getBooleanWithRatherLowProbability()) { // sb.append("DEFAULT"); // } else { @@ -51,7 +51,7 @@ protected void insertValue(DatabendColumn column) { // } String value = DatabendToStringVisitor.asString( new DatabendNoRECExpressionGenerator(globalState). - generateConstant(column.isNullable())); + generateConstant(column.getType().getPrimitiveDataType())); //生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java index c24a9b246..391790804 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -165,14 +165,14 @@ public Node isNull(Node expr) { return null; } - public Node generateConstant(boolean isNullable) { - DatabendDataType type; - do { - type = Randomly.fromOptions(DatabendDataType.values()); - } while(!isNullable && type.equals(DatabendDataType.NULL)); - - return generateConstant(type); - } +// public Node generateConstant(boolean isNullable) { //TODO 极小概率生成NULL值 +// DatabendDataType type; +// do { +// type = Randomly.fromOptions(DatabendDataType.values()); +// } while(!isNullable && type.equals(DatabendDataType.NULL)); //isNullable为假，同时获得NULL就重新获取 +// +// return generateConstant(type); +// } @Override public Node generateConstant(DatabendDataType type) { diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 0dd75375a..6e9eeb57c 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -2,6 +2,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -12,6 +13,7 @@ import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.test.DatabendNoRECOracle; import java.util.ArrayList; import java.util.List; @@ -27,8 +29,8 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(tableName); sb.append("("); List columns = getNewColumns(); - UntypedExpressionGenerator, DatabendColumn> gen = new DatabendExpressionGenerator( - globalState).setColumns(columns); + TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = + new DatabendNoRECExpressionGenerator(globalState).setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); @@ -59,7 +61,8 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { // } if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); - sb.append(DatabendToStringVisitor.asString(gen.generateConstant())); + sb.append(DatabendToStringVisitor.asString( //常量类型于字段类型等同 + gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); sb.append(")"); } } From c2c82b944d4e11d960035a1c8b7d6ae6deb04233 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Thu, 25 Aug 2022 01:16:38 +0800 Subject: [PATCH 173/856] fix: extreme value representation of a double constant --- src/sqlancer/databend/ast/DatabendConstant.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 7b0d09164..56a4661da 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -53,13 +53,12 @@ public double getValue() { @Override public String toString() { if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; + return "3.40282347e+38"; } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; + return "-3.40282347e+38"; } - - return String.valueOf(value).toLowerCase(); // TODO 待databend支持大写E再去掉小写转换 + return String.valueOf(value); } } From 18df9540f4f96eb23a3367b7144f37a343596c0d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Aug 2022 09:09:29 +0000 Subject: [PATCH 174/856] Bump postgresql from 42.4.2 to 42.5.0 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.4.2 to 42.5.0. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.4.2...REL42.5.0) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8d4ce21f1..a6d7d16db 100644 --- a/pom.xml +++ b/pom.xml @@ -282,7 +282,7 @@ org.postgresql postgresql - 42.4.2 + 42.5.0 org.xerial From 7b76fec60ccc43dc8c4eb5c1c3ddc7430993061d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 26 Aug 2022 03:14:00 +0000 Subject: [PATCH 175/856] Bump maven-checkstyle-plugin from 3.1.2 to 3.2.0 Bumps [maven-checkstyle-plugin](https://github.com/apache/maven-checkstyle-plugin) from 3.1.2 to 3.2.0. - [Release notes](https://github.com/apache/maven-checkstyle-plugin/releases) - [Commits](https://github.com/apache/maven-checkstyle-plugin/compare/maven-checkstyle-plugin-3.1.2...maven-checkstyle-plugin-3.2.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-checkstyle-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a6d7d16db..2cd50e527 100644 --- a/pom.xml +++ b/pom.xml @@ -202,7 +202,7 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.1.2 + 3.2.0 com.puppycrawl.tools From b55486b29c2021f59e6274c078b75cbf9294b9c0 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sat, 27 Aug 2022 16:26:31 +0800 Subject: [PATCH 176/856] fix: add the corresponding type name to the original column name --- src/sqlancer/databend/gen/DatabendTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 6e9eeb57c..3fc403157 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -85,8 +85,8 @@ public static String getRandomCollate() { private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - String columnName = String.format("c%d", i); DatabendCompositeDataType columnType = DatabendCompositeDataType.getRandomWithoutNull(); + String columnName = String.format("c%d%s", i,columnType.getPrimitiveDataType().toString()); columns.add(new DatabendColumn(columnName, columnType, false, false)); } return columns; From 8b14856965a9fadaa194926b004be7c72fc8cf79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Aug 2022 09:15:33 +0000 Subject: [PATCH 177/856] Bump checkstyle from 10.3.2 to 10.3.3 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3.2 to 10.3.3. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3.2...checkstyle-10.3.3) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2cd50e527..db142a0d6 100644 --- a/pom.xml +++ b/pom.xml @@ -207,7 +207,7 @@ com.puppycrawl.tools checkstyle - 10.3.2 + 10.3.3 From c8d0e63e3f87e872c2287f035eec1f1889ef2d94 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Mon, 29 Aug 2022 18:08:50 +0800 Subject: [PATCH 178/856] fix: implement where tester of tlp --- .../ast/DatabendUnaryPostfixOperation.java | 4 ++ .../ast/DatabendUnaryPrefixOperation.java | 11 ++-- .../databend/gen/DatabendIndexGenerator.java | 59 ------------------- .../gen/DatabendNoRECExpressionGenerator.java | 53 ++++++++++++++--- .../test/DatabendQueryPartitioningBase.java | 5 +- ...DatabendQueryPartitioningHavingTester.java | 3 +- 6 files changed, 59 insertions(+), 76 deletions(-) delete mode 100644 src/sqlancer/databend/gen/DatabendIndexGenerator.java diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java index e20a6be6b..f516f88a2 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -18,6 +18,10 @@ public DatabendUnaryPostfixOperation(Node expr, DatabendUnar setNegate(negate); } + public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op) { + super(expr,op); + } + public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator { IS_NULL("IS NULL"){ @Override diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index eabb4eabd..64c9328d8 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -17,6 +17,10 @@ public DatabendUnaryPrefixOperation(Node expr, DatabendUnary setNegate(negate); } + public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op) { + super(expr,op); + } + void setNegate(boolean negate){ this.negate = negate; } @@ -26,18 +30,11 @@ public Node getExpression() { return getExpr(); } - @Override - public String getOperatorRepresentation() { - return null; - } - // @Override // public OperatorKind getOperatorKind() { // return OperatorKind.PREFIX; // } - - public enum DatabendUnaryPrefixOperator implements BinaryOperatorNode.Operator { NOT("NOT", DatabendDataType.BOOLEAN, DatabendDataType.INT) { @Override diff --git a/src/sqlancer/databend/gen/DatabendIndexGenerator.java b/src/sqlancer/databend/gen/DatabendIndexGenerator.java deleted file mode 100644 index de6820bcc..000000000 --- a/src/sqlancer/databend/gen/DatabendIndexGenerator.java +++ /dev/null @@ -1,59 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendTable; - -import java.util.List; - -public final class DatabendIndexGenerator { - - private DatabendIndexGenerator() { - } - - public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { - errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); - sb.append("UNIQUE "); - } - sb.append("INDEX "); - sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information - sb.append(" ON "); - DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); - sb.append("("); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(Randomly.fromOptions("ASC", "DESC")); - } - } - sb.append(")"); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - Node expr = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()) - .generateExpression(); - sb.append(DatabendToStringVisitor.asString(expr)); - } - errors.add("already exists!"); - if (globalState.getDbmsSpecificOptions().testRowid) { - errors.add("Cannot create an index on the rowid!"); - } - return new SQLQueryAdapter(sb.toString(), errors, true); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java index 391790804..79939a1bf 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java @@ -1,8 +1,8 @@ package sqlancer.databend.gen; -import com.google.common.collect.RangeMap; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.NewInOperatorNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; @@ -14,7 +14,6 @@ import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -24,6 +23,7 @@ public class DatabendNoRECExpressionGenerator extends TypedExpressionGenerator, DatabendColumn, DatabendDataType> { private final DatabendGlobalState globalState; + private boolean allowAggregateFunctions; public DatabendNoRECExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; @@ -89,7 +89,6 @@ Node generateBooleanExpression(int depth) { Node getPostfix(int depth) { DatabendUnaryPostfixOperator randomOp = DatabendUnaryPostfixOperator.getRandom(); -// System.out.println("getPostfix:" + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth)); return new DatabendUnaryPostfixOperation( generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp,Randomly.getBoolean()); @@ -152,17 +151,17 @@ Node getLike(int depth, DatabendDataType dataType) { @Override public Node generatePredicate() { - return null; + return generateExpression(DatabendDataType.BOOLEAN); } @Override public Node negatePredicate(Node predicate) { - return null; + return new DatabendUnaryPrefixOperation(predicate,DatabendUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return null; + public Node isNull(Node predicate) { + return new DatabendUnaryPostfixOperation(predicate,DatabendUnaryPostfixOperator.IS_NULL); } // public Node generateConstant(boolean isNullable) { //TODO 极小概率生成NULL值 @@ -213,4 +212,44 @@ protected DatabendDataType getRandomType() { protected boolean canGenerateColumnOfType(DatabendDataType type) { return false; } + + public enum DatabendAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1),SUM(1), STDDEV_POP(1), + COVAR_POP(1), COVAR_SAMP(2); + //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) + + private int nrArgs; + + DatabendAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static DatabendAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } + + public NewFunctionNode generateArgsForAggregate( + DatabendAggregateFunction aggregateFunction) { + return new NewFunctionNode( + generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + } + + public Node generateAggregate() { + DatabendAggregateFunction aggrFunc = DatabendAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + public Node generateHavingClause() { + this.allowAggregateFunctions = true; + Node expression = generateExpression(DatabendDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java index a806dd2ca..afa8b2202 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -17,6 +17,7 @@ import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; import java.sql.SQLException; import java.util.ArrayList; @@ -29,7 +30,7 @@ public class DatabendQueryPartitioningBase DatabendSchema s; DatabendTables targetTables; - DatabendExpressionGenerator gen; + DatabendNoRECExpressionGenerator gen; DatabendSelect select; public DatabendQueryPartitioningBase(DatabendGlobalState state) { @@ -50,7 +51,7 @@ public static String canonicalizeResultValue(String value) { public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); - gen = new DatabendExpressionGenerator(state).setColumns(targetTables.getColumns()); + gen = new DatabendNoRECExpressionGenerator(state).setColumns(targetTables.getColumns()); initializeTernaryPredicateVariants(); select = new DatabendSelect(); select.setFetchColumns(generateFetchColumns()); diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java index 019eeace7..9ee9715cd 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -5,6 +5,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendProvider.DatabendGlobalState; @@ -25,7 +26,7 @@ public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } boolean orderBy = Randomly.getBoolean(); if (orderBy) { From 6b14210374413336d9c796462dfd98e843716d35 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 4 Sep 2022 22:53:55 +0800 Subject: [PATCH 179/856] fix: implement all tester of tlp Signed-off-by: Yisong Han --- .../databend/DatabendToStringVisitor.java | 10 ++++++ src/sqlancer/databend/ast/DatabendJoin.java | 4 +-- .../databend/gen/DatabendInsertGenerator.java | 4 +-- ...va => DatabendNewExpressionGenerator.java} | 36 +++++++++++++++---- .../gen/DatabendRandomQuerySynthesizer.java | 2 +- .../databend/gen/DatabendTableGenerator.java | 6 +--- .../databend/test/DatabendNoRECOracle.java | 4 +-- ...abendQueryPartitioningAggregateTester.java | 24 +++++++------ .../test/DatabendQueryPartitioningBase.java | 21 ++++++----- ...DatabendQueryPartitioningHavingTester.java | 16 ++++++--- 10 files changed, 86 insertions(+), 41 deletions(-) rename src/sqlancer/databend/gen/{DatabendNoRECExpressionGenerator.java => DatabendNewExpressionGenerator.java} (88%) diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java index 50a37aeec..628d0502a 100644 --- a/src/sqlancer/databend/DatabendToStringVisitor.java +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -2,6 +2,7 @@ import sqlancer.common.ast.newast.NewToStringVisitor; import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -42,6 +43,15 @@ private void visit(DatabendConstant constant) { sb.append(constant.toString()); } +// private void visitFromList(List> fromList) { +// for (int i = 0; i < fromList.size(); i++) { +// if (i != 0) { +// sb.append(" INNER JOIN "); +// } +// visit(fromList.get(i)); +// } +// } + private void visit(DatabendSelect select) { sb.append("SELECT "); if (select.isDistinct()) { diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index 886f3824c..8797e1508 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -7,7 +7,7 @@ import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; import java.util.ArrayList; import java.util.List; @@ -78,7 +78,7 @@ public static List> getJoins( List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); // DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); - DatabendNoRECExpressionGenerator joinGen = new DatabendNoRECExpressionGenerator(globalState).setColumns(columns); + DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); switch (JoinType.getRandom()) { case INNER: diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index d05f65c14..f0f767d8f 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.databend.gen; -import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -50,7 +48,7 @@ protected void insertValue(DatabendColumn column) { // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); // } String value = DatabendToStringVisitor.asString( - new DatabendNoRECExpressionGenerator(globalState). + new DatabendNewExpressionGenerator(globalState). generateConstant(column.getType().getPrimitiveDataType())); //生成一个与column相同的常量类型 sb.append(value); diff --git a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java similarity index 88% rename from src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java rename to src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 79939a1bf..67d35b06b 100644 --- a/src/sqlancer/databend/gen/DatabendNoRECExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -19,13 +19,13 @@ import java.util.List; import java.util.stream.Collectors; -public class DatabendNoRECExpressionGenerator extends +public class DatabendNewExpressionGenerator extends TypedExpressionGenerator, DatabendColumn, DatabendDataType> { private final DatabendGlobalState globalState; private boolean allowAggregateFunctions; - public DatabendNoRECExpressionGenerator(DatabendGlobalState globalState) { + public DatabendNewExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; } @@ -58,6 +58,14 @@ protected Node generateExpression(DatabendDataType type, int } } + public List> generateExpressions(int nr, DatabendDataType type) { + List> expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(type)); + } + return expressions; + } + private Node generateIntExpression(int depth) { return null; @@ -71,7 +79,7 @@ Node generateBooleanExpression(int depth) { getPostfix(depth + 1); case NOT: getNOT(depth + 1); - case BETWEEN: + case BETWEEN: //TODO (NULL BETWEEN NULL AND NULL) 返回的是 NULL 需要注意 return getBetween(depth + 1); case IN_OPERATION: return getIn(depth + 1); @@ -214,20 +222,35 @@ protected boolean canGenerateColumnOfType(DatabendDataType type) { } public enum DatabendAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1),SUM(1), STDDEV_POP(1), + MAX(1), + MIN(1), + AVG(1,DatabendDataType.INT,DatabendDataType.FLOAT), + COUNT(1), + SUM(1,DatabendDataType.INT,DatabendDataType.FLOAT), + STDDEV_POP(1), COVAR_POP(1), COVAR_SAMP(2); //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) private int nrArgs; + private DatabendDataType[] dataTypes; - DatabendAggregateFunction(int nrArgs) { + DatabendAggregateFunction(int nrArgs, DatabendDataType ...dataTypes) { this.nrArgs = nrArgs; + this.dataTypes = dataTypes; } public static DatabendAggregateFunction getRandom() { return Randomly.fromOptions(values()); } + public DatabendDataType getRandomType() { + if(dataTypes.length == 0) { + return Randomly.fromOptions(DatabendDataType.values()); + } else { + return Randomly.fromOptions(dataTypes); + } + } + public int getNrArgs() { return nrArgs; } @@ -237,7 +260,8 @@ public int getNrArgs() { public NewFunctionNode generateArgsForAggregate( DatabendAggregateFunction aggregateFunction) { return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + generateExpressions(aggregateFunction.getNrArgs(),aggregateFunction.getRandomType()), + aggregateFunction); } public Node generateAggregate() { diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index a0b4c087b..9ac3499b3 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -25,7 +25,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) // .setColumns(targetTables.getColumns()); - DatabendNoRECExpressionGenerator gen = new DatabendNoRECExpressionGenerator(globalState) + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); // TODO distinct diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 3fc403157..ba08f8acc 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -3,21 +3,17 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; -import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; -import sqlancer.databend.test.DatabendNoRECOracle; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; public class DatabendTableGenerator { @@ -30,7 +26,7 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append("("); List columns = getNewColumns(); TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = - new DatabendNoRECExpressionGenerator(globalState).setColumns(columns); + new DatabendNewExpressionGenerator(globalState).setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index d514164ae..451940a6e 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -21,7 +21,7 @@ import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.*; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; -import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; import java.sql.ResultSet; import java.sql.SQLException; @@ -45,7 +45,7 @@ public void check() throws SQLException { DatabendTables randomTables = s.getRandomTableNonEmptyTables(); //随机获得nr张表 List columns = randomTables.getColumns(); // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); - DatabendNoRECExpressionGenerator gen = new DatabendNoRECExpressionGenerator(state).setColumns(columns); + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); //生成随机where条件，形式为ast diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index e584da545..9ce498234 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -23,12 +23,11 @@ import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; -import sqlancer.databend.gen.DatabendExpressionGenerator; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendAggregateFunction; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.gen.DatabendNewExpressionGenerator.DatabendAggregateFunction; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendBinaryArithmeticOperator; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPrefixOperator; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPostfixOperator; import sqlancer.databend.DatabendProvider.DatabendGlobalState; public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { @@ -56,9 +55,9 @@ public void check() throws SQLException { fetchColumns.add(gen.generateAggregate()); } select.setFetchColumns(Arrays.asList(aggregate)); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); - } +// if (Randomly.getBooleanWithRatherLowProbability()) { +// select.setOrderByExpressions(gen.generateOrderBys()); +// } originalQuery = DatabendToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); @@ -80,7 +79,7 @@ public void check() throws SQLException { private String createMetamorphicUnionQuery(DatabendSelect select, NewFunctionNode aggregate, List> from) { String metamorphicQuery; - Node whereClause = gen.generateExpression(); + Node whereClause = gen.generateExpression(DatabendDataType.BOOLEAN); Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, DatabendUnaryPrefixOperator.NOT); Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, @@ -98,7 +97,7 @@ private String createMetamorphicUnionQuery(DatabendSelect select, } private String getAggregateResult(String queryString) throws SQLException { - String resultString; + String resultString = null; SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); try (SQLancerResultSet result = q.executeAndGet(state)) { if (result == null) { @@ -107,7 +106,11 @@ private String getAggregateResult(String queryString) throws SQLException { if (!result.next()) { resultString = null; } else { - resultString = result.getString(1); + try { + resultString = result.getString(1); + } catch (Exception e) { + System.out.println("Invalid integer format for value"); //TODO 超过integer范围无法格式化异常，还未有解决方案 + } } return resultString; } catch (SQLException e) { @@ -180,7 +183,8 @@ private DatabendSelect getSelect(List> aggregates, List leftSelect.setWhereClause(whereClause); leftSelect.setJoinList(joinList); if (Randomly.getBooleanWithSmallProbability()) { - leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); +// leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + leftSelect.setGroupByExpressions(select.getFetchColumns()); } return leftSelect; } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java index afa8b2202..c0d090649 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -12,12 +12,11 @@ import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.gen.DatabendExpressionGenerator; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; -import sqlancer.databend.gen.DatabendNoRECExpressionGenerator; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; import java.sql.SQLException; import java.util.ArrayList; @@ -30,7 +29,7 @@ public class DatabendQueryPartitioningBase DatabendSchema s; DatabendTables targetTables; - DatabendNoRECExpressionGenerator gen; + DatabendNewExpressionGenerator gen; DatabendSelect select; public DatabendQueryPartitioningBase(DatabendGlobalState state) { @@ -51,10 +50,10 @@ public static String canonicalizeResultValue(String value) { public void check() throws SQLException { s = state.getSchema(); targetTables = s.getRandomTableNonEmptyTables(); - gen = new DatabendNoRECExpressionGenerator(state).setColumns(targetTables.getColumns()); + gen = new DatabendNewExpressionGenerator(state).setColumns(targetTables.getColumns()); initializeTernaryPredicateVariants(); select = new DatabendSelect(); - select.setFetchColumns(generateFetchColumns()); + select.setFetchColumns(generateRandomColumns()); List tables = targetTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -66,15 +65,21 @@ public void check() throws SQLException { List> generateFetchColumns() { List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { //TODO 为什么会返回 false 或 true 字段 columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + columns = generateRandomColumns(); } return columns; } + List> generateRandomColumns() { + List> columns; + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + return columns; + } + @Override protected ExpressionGenerator> getGen() { return gen; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java index 9ee9715cd..18c17b452 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -7,6 +7,7 @@ import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendProvider.DatabendGlobalState; @@ -14,6 +15,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Random; public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase implements TestOracle { @@ -28,13 +30,19 @@ public void check() throws SQLException { if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - boolean orderBy = Randomly.getBoolean(); - if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); +// boolean orderBy = Randomly.getBoolean(); + boolean orderBy = false; //关闭order by + if (orderBy) { //TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select +// select.setOrderByExpressions(gen.generateOrderBys()); + List> constants = new ArrayList<>(); + constants.add(new DatabendConstant.DatabendIntConstant(new Random().nextInt(select.getFetchColumns().size()) + 1)); + select.setOrderByExpressions(constants); } - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); +// select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); +// System.out.println(originalQueryString); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setHavingClause(predicate); From 7bd59dc3c9e749f14c11b5cb7cdcf05fe831c3d8 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Mon, 5 Sep 2022 18:17:21 +0800 Subject: [PATCH 180/856] feat(databend): implement binary arithmetic operation Signed-off-by: Yisong Han --- .../DatabendBinaryArithmeticOperation.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java new file mode 100644 index 000000000..3c4caabb9 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -0,0 +1,31 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.Node; + +public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode { + + public DatabendBinaryArithmeticOperation(Node left, Node right, BinaryOperatorNode.Operator op) { + super(left, right, op); + } + + public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Operator{ + ADDITION("+"), + SUBTRACTION("-"), + MULTIPLICATION("*"), + DIVISION("/"), + MODULO("%"); + + DatabendBinaryArithmeticOperator(String text) { + textRepresentation = text; + } + private String textRepresentation; + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + +} From 66a439283298e5a4ea5529ae10a7bfdddaa9c73c Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Mon, 5 Sep 2022 18:17:59 +0800 Subject: [PATCH 181/856] feat(databend): add int expression generator Signed-off-by: Yisong Han --- .../ast/DatabendUnaryPrefixOperation.java | 23 +++++++++ .../databend/gen/DatabendInsertGenerator.java | 2 +- .../gen/DatabendNewExpressionGenerator.java | 51 ++++++++++++------- 3 files changed, 58 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index 64c9328d8..b86bebe6b 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -46,6 +46,29 @@ public DatabendDataType getExpressionType() { protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { return null; // TODO } + }, + + UNARY_PLUS("+",DatabendDataType.INT) { + @Override + public DatabendDataType getExpressionType() { + return DatabendDataType.INT; + } + + @Override + protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { + return expectedValue; + } + }, + UNARY_MINUS("-", DatabendDataType.INT) { + @Override + public DatabendDataType getExpressionType() { + return DatabendDataType.INT; + } + + @Override + protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { + return null; + } }; diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index f0f767d8f..ade2840e4 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -49,7 +49,7 @@ protected void insertValue(DatabendColumn column) { // } String value = DatabendToStringVisitor.asString( new DatabendNewExpressionGenerator(globalState). - generateConstant(column.getType().getPrimitiveDataType())); //生成一个与column相同的常量类型 + generateConstant(column.getType().getPrimitiveDataType(),column.isNullable())); //生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 67d35b06b..9c953cb5e 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -14,6 +14,7 @@ import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -29,11 +30,6 @@ public DatabendNewExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; } - private enum BooleanExpression { - POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; -// SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; - } - public Node generateLeafNode(DatabendDataType dataType) { return generateConstant(dataType); } @@ -49,6 +45,7 @@ protected Node generateExpression(DatabendDataType type, int case BOOLEAN: return generateBooleanExpression(depth); case INT: + return generateIntExpression(depth); case FLOAT: case VARCHAR: case NULL: @@ -66,9 +63,28 @@ public List> generateExpressions(int nr, DatabendDataTy return expressions; } + private enum IntExpression{ + UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION + } + private Node generateIntExpression(int depth) { + IntExpression intExpression = Randomly.fromOptions(IntExpression.values()); + switch (intExpression) { + case UNARY_OPERATION: + return new DatabendUnaryPrefixOperation(generateExpression(DatabendDataType.INT,depth+1), + Randomly.getBoolean()? DatabendUnaryPrefixOperator.UNARY_PLUS : DatabendUnaryPrefixOperator.UNARY_MINUS); + case BINARY_ARITHMETIC_OPERATION: + return new DatabendBinaryArithmeticOperation(generateExpression(DatabendDataType.INT,depth+1), + generateExpression(DatabendDataType.INT,depth+1), + Randomly.fromOptions(DatabendBinaryArithmeticOperator.values())); + default: + throw new AssertionError(); + } + } - return null; + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; +// SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; } Node generateBooleanExpression(int depth) { @@ -172,28 +188,29 @@ public Node isNull(Node predicate) { return new DatabendUnaryPostfixOperation(predicate,DatabendUnaryPostfixOperator.IS_NULL); } -// public Node generateConstant(boolean isNullable) { //TODO 极小概率生成NULL值 -// DatabendDataType type; -// do { -// type = Randomly.fromOptions(DatabendDataType.values()); -// } while(!isNullable && type.equals(DatabendDataType.NULL)); //isNullable为假，同时获得NULL就重新获取 -// -// return generateConstant(type); -// } + public Node generateConstant(DatabendDataType type,boolean isNullable) { + if(isNullable && Randomly.getBooleanWithSmallProbability()) { + createConstant(DatabendDataType.NULL); + } + return createConstant(type); + } @Override public Node generateConstant(DatabendDataType type) { - Randomly r = globalState.getRandomly(); if (Randomly.getBooleanWithSmallProbability()) { return DatabendConstant.createNullConstant(); } + return createConstant(type); + } + public Node createConstant(DatabendDataType type) { + Randomly r = globalState.getRandomly(); switch (type) { case INT: - //不支持string转化故直接返回int constant + //TODO 已支持数值型string转化，待添加 return DatabendConstant.createIntConstant(r.getInteger()); case BOOLEAN: - //不支持string转化故直接返回boolean constant + //TODO 已支持boolean型string转化，待添加 return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); case FLOAT: return DatabendConstant.createFloatConstant((float) r.getDouble()); From 5f07a2dde8134e2937c83228bf8c312359cb953f Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sat, 10 Sep 2022 00:09:46 +0800 Subject: [PATCH 182/856] fix(databend): mvn verify succeed Signed-off-by: Yisong Han --- dependency-reduced-pom.xml | 433 +++++++++++++++++ src/sqlancer/GlobalState.java | 8 +- src/sqlancer/databend/DatabendOptions.java | 18 +- src/sqlancer/databend/DatabendProvider.java | 62 ++- src/sqlancer/databend/DatabendSchema.java | 90 ++-- .../databend/DatabendToStringVisitor.java | 17 +- .../DatabendBinaryArithmeticOperation.java | 14 +- .../DatabendBinaryComparisonOperation.java | 19 +- .../ast/DatabendBinaryLogicalOperation.java | 72 +-- .../databend/ast/DatabendCastOperation.java | 19 + .../databend/ast/DatabendConstant.java | 4 +- src/sqlancer/databend/ast/DatabendJoin.java | 29 +- .../databend/ast/DatabendLikeOperation.java | 11 +- .../ast/DatabendUnaryPostfixOperation.java | 28 +- .../ast/DatabendUnaryPrefixOperation.java | 44 +- .../gen/DatabendAlterTableGenerator.java | 70 --- .../databend/gen/DatabendDeleteGenerator.java | 30 -- .../gen/DatabendExpressionGenerator.java | 451 ------------------ .../databend/gen/DatabendInsertGenerator.java | 26 +- .../gen/DatabendNewExpressionGenerator.java | 215 +++++---- .../gen/DatabendRandomQuerySynthesizer.java | 27 +- .../databend/gen/DatabendTableGenerator.java | 66 +-- .../databend/gen/DatabendUpdateGenerator.java | 48 -- .../databend/gen/DatabendViewGenerator.java | 6 +- .../databend/test/DatabendNoRECOracle.java | 50 +- ...abendQueryPartitioningAggregateTester.java | 42 +- .../test/DatabendQueryPartitioningBase.java | 26 +- ...tabendQueryPartitioningDistinctTester.java | 10 +- ...atabendQueryPartitioningGroupByTester.java | 14 +- ...DatabendQueryPartitioningHavingTester.java | 31 +- .../DatabendQueryPartitioningWhereTester.java | 20 +- .../databend/TestDatabendConnection.java | 10 +- test/sqlancer/dbms/TestClickHouse.java | 2 +- test/sqlancer/dbms/TestDuckDB.java | 2 +- test/sqlancer/dbms/TestMySQLTLP.java | 2 +- 35 files changed, 953 insertions(+), 1063 deletions(-) create mode 100644 dependency-reduced-pom.xml create mode 100644 src/sqlancer/databend/ast/DatabendCastOperation.java delete mode 100644 src/sqlancer/databend/gen/DatabendAlterTableGenerator.java delete mode 100644 src/sqlancer/databend/gen/DatabendDeleteGenerator.java delete mode 100644 src/sqlancer/databend/gen/DatabendExpressionGenerator.java delete mode 100644 src/sqlancer/databend/gen/DatabendUpdateGenerator.java diff --git a/dependency-reduced-pom.xml b/dependency-reduced-pom.xml new file mode 100644 index 000000000..357833ccb --- /dev/null +++ b/dependency-reduced-pom.xml @@ -0,0 +1,433 @@ + + + 4.0.0 + com.sqlancer + sqlancer + SQLancer + 2.0.0 + SQLancer finds logic bugs in Database Management Systems through automatic testing + http://www.sqlancer.com/ + + + mrigger + Manuel Rigger + manuel.rigger@inf.ethz.ch + https://www.manuelrigger.at/ + ETH Zurich + https://ethz.ch/ + + architect + developer + + + + + + MIT License + https://github.com/sqlancer/sqlancer/blob/master/LICENSE.md + repo + + + + scm:git:git://github.com/sqlancer/sqlancer.git + scm:git:ssh://github.com:sqlancer/sqlancer.git + https://github.com/sqlancer/sqlancer/ + + + src + test + + + maven-shade-plugin + 3.3.0 + + + package + + shade + + + + + com.beust:jcommander + + + + + + + + maven-antrun-plugin + 3.1.0 + + + package + + run + + + + + + + + + + + + + maven-surefire-plugin + 2.22.2 + + + org.jacoco + jacoco-maven-plugin + 0.8.8 + + + + prepare-agent + + + + report + test + + report + + + + + + maven-compiler-plugin + 3.10.1 + + + org.codehaus.plexus + plexus-compiler-eclipse + 2.12.0 + + + org.eclipse.jdt + ecj + 3.28.0 + + + org.codehaus.plexus + plexus-compiler-api + 2.12.0 + + + + 11 + 11 + + ${project.basedir}/.settings/org.eclipse.jdt.core.prefs + + eclipse + true + true + + + + net.revelc.code.formatter + formatter-maven-plugin + 2.19.0 + + + eclipseformat + verify + + validate + + + + + + maven-dependency-plugin + 3.2.0 + + + copy-dependencies + prepare-package + + copy-dependencies + + + ${project.build.directory}/lib + false + false + true + jcommander + + + + + + maven-jar-plugin + 3.2.2 + + true + + + true + lib/ + sqlancer.Main + + + + + + org.codehaus.mojo + xml-maven-plugin + 1.0.2 + + + xml-check-format + verify + + check-format + + + + + + maven-checkstyle-plugin + 3.1.2 + + + checkstyle + verify + + check + + + true + + + + + + com.puppycrawl.tools + checkstyle + 10.3 + + + + configs/checkstyle.xml + + + + maven-pmd-plugin + 3.14.0 + + + pmd + verify + + check + + + 2 + true + + configs/pmd-rules.xml + + + + + + + com.github.spotbugs + spotbugs-maven-plugin + 4.7.0.0 + + + spotbugs + verify + + check + + + configs/spotbugs-exclude.xml + High + + + + + + + + + jdk-8-config + + ${java.home}/../bin/javadoc + + + + jdk-11-config + + ${java.home}/bin/javadoc + + + + release-steps + + + + maven-source-plugin + 3.2.1 + + + attach-sources + + jar-no-fork + + + + + + maven-javadoc-plugin + 3.4.0 + + + attach-javadocs + + jar + + + + + 8 + + + + maven-gpg-plugin + 3.0.1 + + + sign-artifacts + verify + + sign + + + + --pinentry-mode + loopback + + + + + + + + + + + + com.google.auto.service + auto-service + 1.0.1 + compile + + + org.postgresql + postgresql + 42.4.0 + compile + + + org.xerial + sqlite-jdbc + 3.36.0.3 + compile + + + mysql + mysql-connector-java + 8.0.29 + compile + + + org.mariadb.jdbc + mariadb-java-client + 3.0.5 + compile + + + org.duckdb + duckdb_jdbc + 0.4.0 + compile + + + org.junit.jupiter + junit-jupiter-engine + 5.8.2 + test + + + junit-platform-engine + org.junit.platform + + + junit-jupiter-api + org.junit.jupiter + + + apiguardian-api + org.apiguardian + + + + + org.slf4j + slf4j-simple + 1.7.36 + compile + + + ru.yandex.clickhouse + clickhouse-jdbc + 0.3.2 + compile + + + com.h2database + h2 + 2.1.214 + compile + + + org.mongodb + mongodb-driver-sync + 4.1.1 + compile + + + com.arangodb + arangodb-java-driver + 6.9.0 + compile + + + + + + maven-jxr-plugin + 3.2.0 + + + + + + ossrh + Central Repository OSSRH + https://oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + UTF-8 + + diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 20efcdcc8..9c48dd900 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,12 +1,12 @@ package sqlancer; +import java.sql.SQLException; + import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; -import java.sql.SQLException; - public abstract class GlobalState, S extends AbstractSchema, C extends SQLancerDBConnection> { protected C databaseConnection; @@ -144,10 +144,10 @@ protected void setSchema(S schema) { } public void updateSchema() throws Exception { - try{ + try { setSchema(readSchema()); } catch (SQLException sqlException) { - System.out.println(String.format("%s readSchema SQLException",databaseName)); + System.out.printf("%s readSchema SQLException%n", databaseName); sqlException.printStackTrace(); } for (AbstractTable table : schema.getDatabaseTables()) { diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index de6940a76..ebacd21cf 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -1,19 +1,25 @@ package sqlancer.databend; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; + import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.databend.test.*; import sqlancer.databend.DatabendOptions.DatabendOracleFactory; -import sqlancer.databend.test.DatabendQueryPartitioningAggregateTester; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; +import sqlancer.databend.test.DatabendNoRECOracle; +import sqlancer.databend.test.DatabendQueryPartitioningAggregateTester; +import sqlancer.databend.test.DatabendQueryPartitioningDistinctTester; +import sqlancer.databend.test.DatabendQueryPartitioningGroupByTester; +import sqlancer.databend.test.DatabendQueryPartitioningHavingTester; +import sqlancer.databend.test.DatabendQueryPartitioningWhereTester; @Parameters(commandDescription = "Databend") public class DatabendOptions implements DBMSSpecificOptions { diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index dd54a56db..793650798 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -1,19 +1,30 @@ package sqlancer.databend; -import com.google.auto.service.AutoService; -import sqlancer.*; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLQueryProvider; -import sqlancer.databend.gen.*; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; - import java.io.File; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.gen.DatabendInsertGenerator; +import sqlancer.databend.gen.DatabendRandomQuerySynthesizer; +import sqlancer.databend.gen.DatabendTableGenerator; + @AutoService(DatabaseProvider.class) public class DatabendProvider extends SQLProviderAdapter { @@ -24,10 +35,11 @@ public DatabendProvider() { public enum Action implements AbstractAction { INSERT(DatabendInsertGenerator::getQuery), // - //TODO 等待databend实现update && delete -// DELETE(DatabendDeleteGenerator::generate), // -// UPDATE(DatabendUpdateGenerator::getQuery), // -// CREATE_VIEW(DatabendViewGenerator::generate), // TODO 等待databend的create view语法 更加贴近mysql + // TODO 等待databend实现update && delete + // DELETE(DatabendDeleteGenerator::generate), // + // UPDATE(DatabendUpdateGenerator::getQuery), // + + // CREATE_VIEW(DatabendViewGenerator::generate), //TODO 等待databend的create view语法 更加贴近mysql EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); @@ -57,13 +69,13 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case EXPLAIN: return r.getInteger(0, 2); - //TODO 等待databend实现update && delete -// case UPDATE: -// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); -// case DELETE: -// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); -// case CREATE_VIEW: -// return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + // TODO 等待databend实现update && delete + // case UPDATE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + // case DELETE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + // case CREATE_VIEW: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); } @@ -96,7 +108,7 @@ public void generateDatabase(DatabendGlobalState globalState) throws Exception { throw new IgnoreMeException(); } }); - se.executeStatements(); //在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 + se.executeStatements(); // 在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 } public void tryDeleteFile(String fname) { @@ -144,17 +156,17 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE globalState.getState().logStatement("USE " + databaseName); } -// try (Statement s = con.createStatement()) { -// s.execute("set enable_planner_v2 = 0;"); -// globalState.getState().logStatement("set enable_planner_v2 = 0;"); -// } + // try (Statement s = con.createStatement()) { + // s.execute("set enable_planner_v2 = 0;"); + // globalState.getState().logStatement("set enable_planner_v2 = 0;"); + // } return new SQLConnection(con); } @Override public String getDBMSName() { - return "databend"; //用于DatabendOptions + return "databend"; // 用于DatabendOptions } } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index ad92932d2..643f53dd9 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -1,13 +1,5 @@ package sqlancer.databend; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.DBMSCommon; -import sqlancer.common.schema.*; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendTable; - import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -15,12 +7,24 @@ import java.util.Collections; import java.util.List; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendTable; + public class DatabendSchema extends AbstractSchema { public enum DatabendDataType { INT, VARCHAR, BOOLEAN, FLOAT, NULL; - //, DATE, TIMESTAMP + // , DATE, TIMESTAMP public static DatabendDataType getRandomWithoutNull() { DatabendDataType dt; @@ -74,8 +78,8 @@ public static DatabendCompositeDataType getRandomWithoutNull() { break; case BOOLEAN: case VARCHAR: -// case DATE: -// case TIMESTAMP: + // case DATE: + // case TIMESTAMP: size = 0; break; default: @@ -114,10 +118,10 @@ public String toString() { } case BOOLEAN: return Randomly.fromOptions("BOOLEAN", "BOOL"); -// case TIMESTAMP: -// return Randomly.fromOptions("TIMESTAMP", "DATETIME"); -// case DATE: -// return Randomly.fromOptions("DATE"); + // case TIMESTAMP: + // return Randomly.fromOptions("TIMESTAMP", "DATETIME"); + // case DATE: + // return Randomly.fromOptions("DATE"); case NULL: return Randomly.fromOptions("NULL"); default: @@ -132,7 +136,8 @@ public static class DatabendColumn extends AbstractTableColumn columns, boolean isV public static DatabendSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); - List tableNames = getTableNames(con,databaseName); + List tableNames = getTableNames(con, databaseName); for (String tableName : tableNames) { if (DBMSCommon.matchesIndexName(tableName)) { continue; // TODO: unexpected? } - List databaseColumns = getTableColumns(con, tableName,databaseName); + List databaseColumns = getTableColumns(con, tableName, databaseName); boolean isView = tableName.startsWith("v"); DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); for (DatabendColumn c : databaseColumns) { @@ -250,18 +255,20 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { List tableNames = null; tableNames = new ArrayList<>(); - //SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema != 'system' and table_schema != 'INFORMATION_SCHEMA' and table_type='BASE TABLE' - //"SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName + // SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema != 'system' and table_schema != + // 'INFORMATION_SCHEMA' and table_type='BASE TABLE' + // "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName final String sqlStatement = String.format( - "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName); + "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ", + databaseName); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(sqlStatement)) { - try{ //没有catch的话rs.next()会报SQLException：Not a navigable ResultSet + try { // 没有catch的话rs.next()会报SQLException：Not a navigable ResultSet while (rs.next()) { tableNames.add(rs.getString("table_name")); } - } catch (Exception e){ -// e.printStackTrace(); + } catch (Exception e) { + // e.printStackTrace(); System.out.println("TableNames->SQLException：Not a navigable ResultSet"); } } @@ -269,20 +276,22 @@ private static List getTableNames(SQLConnection con, String databaseName return tableNames; } - private static List getTableColumns(SQLConnection con, String tableName, String databaseName) throws SQLException { + private static List getTableColumns(SQLConnection con, String tableName, String databaseName) + throws SQLException { List columns = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format( "SELECT * FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = '%s' and table_name ='%s'", - databaseName,tableName))) { - try{ //没有catch的话rs.next()会报SQLException：Not a navigable ResultSet + databaseName, tableName))) { + try { // 没有catch的话rs.next()会报SQLException：Not a navigable ResultSet while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); boolean isNullable = rs.getBoolean("is_nullable"); -// boolean isPrimaryKey = rs.getString("pk").contains("true"); - boolean isPrimaryKey = false; //没找到主键元数据 - DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); + // boolean isPrimaryKey = rs.getString("pk").contains("true"); + boolean isPrimaryKey = false; // 没找到主键元数据 + DatabendColumn c = new DatabendColumn(columnName, getColumnType(dataType), isPrimaryKey, + isNullable); columns.add(c); } } catch (Exception e) { @@ -290,10 +299,11 @@ private static List getTableColumns(SQLConnection con, String ta } } } - if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { - // TODO: implement an option to enable/disable rowids -// columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, false)); - } + // if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { + // TODO: implement an option to enable/disable rowids + // columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, + // false)); + // } return columns; } diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java index 628d0502a..b3669e4b7 100644 --- a/src/sqlancer/databend/DatabendToStringVisitor.java +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -2,7 +2,6 @@ import sqlancer.common.ast.newast.NewToStringVisitor; import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -43,14 +42,14 @@ private void visit(DatabendConstant constant) { sb.append(constant.toString()); } -// private void visitFromList(List> fromList) { -// for (int i = 0; i < fromList.size(); i++) { -// if (i != 0) { -// sb.append(" INNER JOIN "); -// } -// visit(fromList.get(i)); -// } -// } + // private void visitFromList(List> fromList) { + // for (int i = 0; i < fromList.size(); i++) { + // if (i != 0) { + // sb.append(" INNER JOIN "); + // } + // visit(fromList.get(i)); + // } + // } private void visit(DatabendSelect select) { sb.append("SELECT "); diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java index 3c4caabb9..dfc44ee22 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -6,21 +6,19 @@ public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode { - public DatabendBinaryArithmeticOperation(Node left, Node right, BinaryOperatorNode.Operator op) { + public DatabendBinaryArithmeticOperation(Node left, Node right, + BinaryOperatorNode.Operator op) { super(left, right, op); } - public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Operator{ - ADDITION("+"), - SUBTRACTION("-"), - MULTIPLICATION("*"), - DIVISION("/"), - MODULO("%"); + public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + ADDITION("+"), SUBTRACTION("-"), MULTIPLICATION("*"), DIVISION("/"), MODULO("%"); + + private final String textRepresentation; DatabendBinaryArithmeticOperator(String text) { textRepresentation = text; } - private String textRepresentation; @Override public String getTextRepresentation() { diff --git a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java index f76579ca3..9e8e43a51 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java @@ -4,28 +4,24 @@ import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.Node; - public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode { public DatabendBinaryComparisonOperation(Node left, Node right, - DatabendBinaryComparisonOperator op) { + DatabendBinaryComparisonOperator op) { super(left, right, op); } - public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Operator{ + public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Operator { EQUALS("="), -// IS_DISTINCT("IS DISTINCT FROM"), -// IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), - NOT_EQUALS("!="), - LESS("<"), - LESS_EQUALS("<="), - GREATER(">"), - GREATER_EQUALS(">="); + // IS_DISTINCT("IS DISTINCT FROM"), + // IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), + NOT_EQUALS("!="), LESS("<"), LESS_EQUALS("<="), GREATER(">"), GREATER_EQUALS(">="); + + private final String textRepresentation; DatabendBinaryComparisonOperator(String text) { textRepresentation = text; } - private String textRepresentation; @Override public String getTextRepresentation() { @@ -33,5 +29,4 @@ public String getTextRepresentation() { } } - } diff --git a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java index 32c8df0ca..14ff984e6 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java @@ -8,47 +8,47 @@ public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode { public DatabendBinaryLogicalOperation(Node left, Node right, - DatabendBinaryLogicalOperator op) { - super(left,right,op); + DatabendBinaryLogicalOperator op) { + super(left, right, op); } public enum DatabendBinaryLogicalOperator implements BinaryOperatorNode.Operator { AND("AND", "and") { -// @Override -// public DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right) { -// if (left.isNull() && right.isNull()) { -// return DatabendNoRECConstant.createNullConstant(); -// } else if (left.isNull()) { -// if (right.asBooleanNotNull()) { -// return DatabendNoRECConstant.createNullConstant(); -// } else { -// return DatabendNoRECConstant.createFalse(); -// } -// } else if (right.isNull()) { -// if (left.asBooleanNotNull()) { -// return DatabendNoRECConstant.createNullConstant(); -// } else { -// return DatabendNoRECConstant.createFalse(); -// } -// } else { -// return left.asBooleanNotNull() && right.asBooleanNotNull() ? DatabendNoRECConstant.createTrue() -// : DatabendNoRECConstant.createFalse(); -// } -// } + // @Override + // public DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right) { + // if (left.isNull() && right.isNull()) { + // return DatabendNoRECConstant.createNullConstant(); + // } else if (left.isNull()) { + // if (right.asBooleanNotNull()) { + // return DatabendNoRECConstant.createNullConstant(); + // } else { + // return DatabendNoRECConstant.createFalse(); + // } + // } else if (right.isNull()) { + // if (left.asBooleanNotNull()) { + // return DatabendNoRECConstant.createNullConstant(); + // } else { + // return DatabendNoRECConstant.createFalse(); + // } + // } else { + // return left.asBooleanNotNull() && right.asBooleanNotNull() ? DatabendNoRECConstant.createTrue() + // : DatabendNoRECConstant.createFalse(); + // } + // } }, OR("OR", "or") { -// @Override -// public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { -// if (!left.isNull() && left.asBooleanNotNull()) { -// return ClickHouseConstant.createTrue(); -// } else if (!right.isNull() && right.asBooleanNotNull()) { -// return ClickHouseConstant.createTrue(); -// } else if (left.isNull() || right.isNull()) { -// return ClickHouseConstant.createNullConstant(); -// } else { -// return ClickHouseConstant.createFalse(); -// } -// } + // @Override + // public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { + // if (!left.isNull() && left.asBooleanNotNull()) { + // return ClickHouseConstant.createTrue(); + // } else if (!right.isNull() && right.asBooleanNotNull()) { + // return ClickHouseConstant.createTrue(); + // } else if (left.isNull() || right.isNull()) { + // return ClickHouseConstant.createNullConstant(); + // } else { + // return ClickHouseConstant.createFalse(); + // } + // } }; private final String[] textRepresentations; @@ -66,7 +66,7 @@ public DatabendBinaryLogicalOperator getRandomOp() { return Randomly.fromOptions(values()); } -// public abstract DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right); + // public abstract DatabendNoRECConstant apply(DatabendNoRECConstant left, DatabendNoRECConstant right); public static DatabendBinaryLogicalOperator getRandom() { return Randomly.fromOptions(values()); diff --git a/src/sqlancer/databend/ast/DatabendCastOperation.java b/src/sqlancer/databend/ast/DatabendCastOperation.java new file mode 100644 index 000000000..c1b3ffe66 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendCastOperation.java @@ -0,0 +1,19 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendSchema; + +public class DatabendCastOperation extends NewUnaryPostfixOperatorNode { + + public DatabendCastOperation(Node expr, DatabendSchema.DatabendCompositeDataType type) { + super(expr, new BinaryOperatorNode.Operator() { + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 56a4661da..39056aff9 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -1,10 +1,10 @@ package sqlancer.databend.ast; -import sqlancer.common.ast.newast.Node; - import java.sql.Timestamp; import java.text.SimpleDateFormat; +import sqlancer.common.ast.newast.Node; + public class DatabendConstant implements Node { private DatabendConstant() { diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index 8797e1508..8694608cd 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -1,17 +1,17 @@ package sqlancer.databend.ast; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -import java.util.ArrayList; -import java.util.List; - public class DatabendJoin implements Node { private final TableReferenceNode leftTable; @@ -37,8 +37,8 @@ public static OuterType getRandom() { } public DatabendJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; @@ -77,8 +77,9 @@ public static List> getJoins( TableReferenceNode rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); -// DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); - DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + // DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); + DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState) + .setColumns(columns); switch (JoinType.getRandom()) { case INNER: @@ -90,11 +91,11 @@ public static List> getJoins( break; case LEFT: joinExpressions.add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case RIGHT: joinExpressions.add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; default: throw new AssertionError(); @@ -104,22 +105,22 @@ public static List> getJoins( } public static DatabendJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new DatabendJoin(left, right, JoinType.RIGHT, predicate); } public static DatabendJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new DatabendJoin(left, right, JoinType.LEFT, predicate); } public static DatabendJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new DatabendJoin(left, right, JoinType.INNER, predicate); } public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, OuterType naturalJoinType) { + TableReferenceNode right, OuterType naturalJoinType) { DatabendJoin join = new DatabendJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; diff --git a/src/sqlancer/databend/ast/DatabendLikeOperation.java b/src/sqlancer/databend/ast/DatabendLikeOperation.java index e76219b1a..2755970e7 100644 --- a/src/sqlancer/databend/ast/DatabendLikeOperation.java +++ b/src/sqlancer/databend/ast/DatabendLikeOperation.java @@ -7,19 +7,18 @@ public class DatabendLikeOperation extends NewBinaryOperatorNode { - public DatabendLikeOperation(Node left, Node right - , DatabendLikeOperator op) { + public DatabendLikeOperation(Node left, Node right, + DatabendLikeOperator op) { super(left, right, op); } - public enum DatabendLikeOperator implements BinaryOperatorNode.Operator { - LIKE_OPERATOR("LIKE","like"); + LIKE_OPERATOR("LIKE", "like"); private final String[] textRepresentations; - DatabendLikeOperator(String... text){ - textRepresentations = text; + DatabendLikeOperator(String... text) { + textRepresentations = text.clone(); } @Override diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java index f516f88a2..e01c35282 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -6,36 +6,36 @@ import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendSchema.DatabendDataType; - public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { -// private final Node expr; -// private final DatabendUnaryPostfixOperator op; + // private final Node expr; + // private final DatabendUnaryPostfixOperator op; private boolean negate; - public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op, boolean negate) { - super(expr,op); + public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op, + boolean negate) { + super(expr, op); setNegate(negate); } public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op) { - super(expr,op); + super(expr, op); } public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator { - IS_NULL("IS NULL"){ + IS_NULL("IS NULL") { @Override public DatabendDataType[] getInputDataTypes() { return DatabendDataType.values(); } }, - IS_NOT_NULL("IS NOT NULL"){ + IS_NOT_NULL("IS NOT NULL") { @Override public DatabendDataType[] getInputDataTypes() { return DatabendDataType.values(); } }; - //IS + // IS private final String textRepresentations; @@ -64,7 +64,7 @@ public void setNegate(boolean negate) { this.negate = negate; } -// @Override + // @Override public Node getExpression() { return getExpr(); } @@ -74,8 +74,8 @@ public String getOperatorRepresentation() { return this.op.getTextRepresentation(); } -// @Override -// public OperatorKind getOperatorKind() { -// return OperatorKind.POSTFIX; -// } + // @Override + // public OperatorKind getOperatorKind() { + // return OperatorKind.POSTFIX; + // } } diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index b86bebe6b..69c95a745 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -1,39 +1,40 @@ package sqlancer.databend.ast; +import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendSchema.DatabendDataType; - public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { -// private final Node expr; -// private final DatabendUnaryPrefixOperator op; - private boolean negate; - - public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op, boolean negate) { - super(expr,op); - setNegate(negate); - } + // private final Node expr; + // private final DatabendUnaryPrefixOperator op; + // private boolean negate; public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op) { - super(expr,op); + super(expr, op); } - void setNegate(boolean negate){ - this.negate = negate; - } + // public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op, boolean + // negate) { + // super(expr,op); + // setNegate(negate); + // } + + // void setNegate(boolean negate){ + // this.negate = negate; + // } -// @Override + // @Override public Node getExpression() { return getExpr(); } -// @Override -// public OperatorKind getOperatorKind() { -// return OperatorKind.PREFIX; -// } + // @Override + // public OperatorKind getOperatorKind() { + // return OperatorKind.PREFIX; + // } public enum DatabendUnaryPrefixOperator implements BinaryOperatorNode.Operator { NOT("NOT", DatabendDataType.BOOLEAN, DatabendDataType.INT) { @@ -48,7 +49,7 @@ protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { } }, - UNARY_PLUS("+",DatabendDataType.INT) { + UNARY_PLUS("+", DatabendDataType.INT) { @Override public DatabendDataType getExpressionType() { return DatabendDataType.INT; @@ -71,7 +72,6 @@ protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { } }; - private String textRepresentation; private DatabendDataType[] dataTypes; @@ -82,8 +82,8 @@ protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { public abstract DatabendDataType getExpressionType(); - public DatabendDataType[] getInputDataTypes(){ - return dataTypes; + public DatabendDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); } protected abstract DatabendConstant getExpectedValue(DatabendConstant expectedValue); diff --git a/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java b/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java deleted file mode 100644 index 90920c819..000000000 --- a/src/sqlancer/databend/gen/DatabendAlterTableGenerator.java +++ /dev/null @@ -1,70 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; -import sqlancer.databend.DatabendSchema.DatabendTable; - -public final class DatabendAlterTableGenerator { - - private DatabendAlterTableGenerator() { - } - - enum Action { - ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN - } - - public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - errors.add(" does not have a column with name \"rowid\""); - errors.add("Table does not contain column rowid referenced in alter statement"); - StringBuilder sb = new StringBuilder("ALTER TABLE "); - DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()); - sb.append(table.getName()); - sb.append(" "); - Action action = Randomly.fromOptions(Action.values()); - switch (action) { - case ADD_COLUMN: - sb.append("ADD COLUMN "); - String columnName = table.getFreeColumnName(); - sb.append(columnName); - sb.append(" "); - sb.append(DatabendCompositeDataType.getRandomWithoutNull().toString()); - break; - case ALTER_COLUMN: - sb.append("ALTER COLUMN "); - sb.append(table.getRandomColumn().getName()); - sb.append(" SET DATA TYPE "); - sb.append(DatabendCompositeDataType.getRandomWithoutNull().toString()); - if (Randomly.getBoolean()) { - sb.append(" USING "); - DatabendErrors.addExpressionErrors(errors); - sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); - } - errors.add("Cannot change the type of this column: an index depends on it!"); - errors.add("Cannot change the type of a column that has a UNIQUE or PRIMARY KEY constraint specified"); - errors.add("Unimplemented type for cast"); - errors.add("Conversion:"); - errors.add("Cannot change the type of a column that has a CHECK constraint specified"); - break; - case DROP_COLUMN: - sb.append("DROP COLUMN "); - sb.append(table.getRandomColumn().getName()); - errors.add("named in key does not exist"); // TODO - errors.add("Cannot drop this column:"); - errors.add("Cannot drop column: table only has one column remaining!"); - errors.add("because there is a CHECK constraint that depends on it"); - errors.add("because there is a UNIQUE constraint that depends on it"); - break; - default: - throw new AssertionError(action); - } - return new SQLQueryAdapter(sb.toString(), errors, true); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java deleted file mode 100644 index 25dd83f45..000000000 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ /dev/null @@ -1,30 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendTable; - -public final class DatabendDeleteGenerator { - - private DatabendDeleteGenerator() { - } - - public static SQLQueryAdapter generate(DatabendGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); - DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DatabendToStringVisitor.asString( - new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); - } - DatabendErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendExpressionGenerator.java deleted file mode 100644 index a1a0fd3d9..000000000 --- a/src/sqlancer/databend/gen/DatabendExpressionGenerator.java +++ /dev/null @@ -1,451 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.*; -import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; -import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.databend.ast.DatabendConstant; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; -import sqlancer.databend.DatabendSchema.DatabendDataType; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public final class DatabendExpressionGenerator extends UntypedExpressionGenerator, DatabendColumn> { - - private final DatabendGlobalState globalState; - - public DatabendExpressionGenerator(DatabendGlobalState globalState) { - this.globalState = globalState; - } - - private enum Expression { - UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL, BETWEEN, IN - // COLLATE, LIKE_ESCAPE, CAST, CASE - //TODO 数学运算时去掉Boolean，或等待databend允许boolean参与数学运算 - //FUNC(数学函数不支持boolean),BINARY_ARITHMETIC(不支持boolean参与二元运算 - - } - - private enum BooleanExpression { - UNARY_POSTFIX, BINARY_COMPARISON,BINARY_LOGICAL - } - - @Override - protected Node generateExpression(int depth) { -// if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { -// return generateLeafNode(); -// } - if (depth >= 1) { - return generateLeafNode(); - } -// if (allowAggregates && Randomly.getBoolean()) { -// DatabendAggregateFunction aggregate = DatabendAggregateFunction.getRandom(); -// allowAggregates = false; -// return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); -// } - List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); -// if (!globalState.getDbmsSpecificOptions().testCollate) { -// possibleOptions.remove(Expression.COLLATE); -// } -// if (!globalState.getDbmsSpecificOptions().testFunctions) { -// possibleOptions.remove(Expression.FUNC); -// } -// if (!globalState.getDbmsSpecificOptions().testCasts) { -// possibleOptions.remove(Expression.CAST); -// } - if (!globalState.getDbmsSpecificOptions().testBetween) { - possibleOptions.remove(Expression.BETWEEN); - } - if (!globalState.getDbmsSpecificOptions().testIn) { - possibleOptions.remove(Expression.IN); - } -// if (!globalState.getDbmsSpecificOptions().testCase) { -// possibleOptions.remove(Expression.CASE); -// } - if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { - possibleOptions.remove(Expression.BINARY_COMPARISON); - } - if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { - possibleOptions.remove(Expression.BINARY_LOGICAL); - } - Expression expr = Randomly.fromList(possibleOptions); - switch (expr) { -// case COLLATE: -// return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), -// DatabendCollate.getRandom()); - case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), - DatabendUnaryPrefixOperator.getRandom()); - case UNARY_POSTFIX: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), - DatabendUnaryPostfixOperator.getRandom()); - case BINARY_COMPARISON: - Operator op = DatabendBinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); - case BINARY_LOGICAL: - op = DatabendBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); -// case BINARY_ARITHMETIC: -// return new NewBinaryOperatorNode(generateExpression(depth + 1), -// generateExpression(depth + 1), DatabendBinaryArithmeticOperator.getRandom()); -// case CAST: -// return new DatabendCastOperation(generateExpression(depth + 1), -// DatabendCompositeDataType.getRandomWithoutNull()); -// case FUNC: -// DBFunction func = DBFunction.getRandom(); -// return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); - case BETWEEN: - return new NewBetweenOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); - case IN: - return new NewInOperatorNode(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); -// case CASE: -// int nr = Randomly.smallNumber() + 1; -// return new NewCaseOperatorNode(generateExpression(depth + 1), -// generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), -// generateExpression(depth + 1)); -// case LIKE_ESCAPE: -// return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), -// generateExpression(depth + 1), "LIKE", "ESCAPE"); - default: - System.out.println(expr.toString()); - throw new AssertionError(); - } - } - - @Override - protected Node generateColumn() { - DatabendColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode(column); - } - - @Override - public Node generateConstant() { - if (Randomly.getBooleanWithSmallProbability()) { - return DatabendConstant.createNullConstant(); - } -// DatabendDataType type = DatabendDataType.getRandomWithoutNull(); - //TODO 先跳过varchar等待databend对其更好的支持，或自己改写case VARCHAR的代码 - DatabendDataType type = DatabendDataType.getRandomWithoutNullAndVarchar(); - switch (type) { - case INT: - if (!globalState.getDbmsSpecificOptions().testIntConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createIntConstant(globalState.getRandomly().getInteger()); -// case DATE: -// if (!globalState.getDbmsSpecificOptions().testDateConstants) { -// throw new IgnoreMeException(); -// } -// return DatabendConstant.createDateConstant(globalState.getRandomly().getInteger()); -// case TIMESTAMP: -// if (!globalState.getDbmsSpecificOptions().testTimestampConstants) { -// throw new IgnoreMeException(); -// } -// return DatabendConstant.createTimestampConstant(globalState.getRandomly().getInteger()); - case VARCHAR: - if (!globalState.getDbmsSpecificOptions().testStringConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createStringConstant(globalState.getRandomly().getString()); - case BOOLEAN: - if (!globalState.getDbmsSpecificOptions().testBooleanConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); - case FLOAT: - if (!globalState.getDbmsSpecificOptions().testFloatConstants) { - throw new IgnoreMeException(); - } - return DatabendConstant.createFloatConstant(globalState.getRandomly().getDouble()); - default: - throw new AssertionError(); - } - } - - @Override - public List> generateOrderBys() { - List> expr = super.generateOrderBys(); - List> newExpr = new ArrayList<>(expr.size()); - for (Node curExpr : expr) { - if (Randomly.getBoolean()) { - curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); - } - newExpr.add(curExpr); - } - return newExpr; - }; - - public static class DatabendCastOperation extends NewUnaryPostfixOperatorNode { - - public DatabendCastOperation(Node expr, DatabendCompositeDataType type) { - super(expr, new Operator() { - - @Override - public String getTextRepresentation() { - return "::" + type.toString(); - } - }); - } - - } - - public enum DatabendAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1),SUM(1), STDDEV_POP(1), - COVAR_POP(1), COVAR_SAMP(2); - //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) - - private int nrArgs; - - DatabendAggregateFunction(int nrArgs) { - this.nrArgs = nrArgs; - } - - public static DatabendAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - - } - - public enum DBFunction { - // trigonometric functions, nrArgs表示参数个数 - ACOS(1), // - ASIN(1), // - ATAN(1), // - COS(1), // - SIN(1), // - TAN(1), // - COT(1), // - ATAN2(2), // - // math functions - ABS(1), // - CEIL(1), // - CEILING(1), // - FLOOR(1), // - LOG(1), // - LOG10(1), LOG2(1), // - LN(1), // - PI(0), // - SQRT(1), // - POWER(1), // - CBRT(1), // - ROUND(2), // - SIGN(1), // - DEGREES(1), // - RADIANS(1), // - MOD(2), // - XOR(2), // - // string functions - LENGTH(1), // - LOWER(1), // - UPPER(1), // - SUBSTRING(3), // - REVERSE(1), // - CONCAT(1, true), // - CONCAT_WS(1, true), CONTAINS(2), // - PREFIX(2), // - SUFFIX(2), // - INSTR(2), // - PRINTF(1, true), // - REGEXP_MATCHES(2), // - REGEXP_REPLACE(3), // - STRIP_ACCENTS(1), // - - // date functions - DATE_PART(2), AGE(2), - - COALESCE(3), NULLIF(2), - - // LPAD(3), - // RPAD(3), - LTRIM(1), RTRIM(1), - // LEFT(2), https://github.com/cwida/Databend/issues/633 - // REPEAT(2), - REPLACE(3), UNICODE(1), - - BIT_COUNT(1), BIT_LENGTH(1), MONTHNAME(1), DAYNAME(1), YEARWEEK(1), DAYOFMONTH(1), WEEKDAY(1), - WEEKOFYEAR(1), - //, LAST_DAY(1) - - IFNULL(2), IF(3); - - private int nrArgs; - private boolean isVariadic; - - DBFunction(int nrArgs) { - this(nrArgs, false); - } - - DBFunction(int nrArgs, boolean isVariadic) { - this.nrArgs = nrArgs; - this.isVariadic = isVariadic; - } - - public static DBFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - if (isVariadic) { - return Randomly.smallNumber() + nrArgs; - } else { - return nrArgs; - } - } - - } - - public enum DatabendUnaryPostfixOperator implements Operator { - - IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); - - private String textRepr; - - DatabendUnaryPostfixOperator(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - public static DatabendUnaryPostfixOperator getRandom() { - return Randomly.fromOptions(values()); - } - - } - - public static final class DatabendCollate implements Operator { - - private final String textRepr; - - private DatabendCollate(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return "COLLATE " + textRepr; - } - - public static DatabendCollate getRandom() { - return new DatabendCollate(DatabendTableGenerator.getRandomCollate()); - } - - } - - public enum DatabendUnaryPrefixOperator implements Operator { - - NOT("NOT"); - //PLUS("+"), MINUS("-"); //等待boolean支持数学运算 - - private String textRepr; - - DatabendUnaryPrefixOperator(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - public static DatabendUnaryPrefixOperator getRandom() { - return Randomly.fromOptions(values()); - } - - } - - public enum DatabendBinaryLogicalOperator implements Operator { - - AND, OR; - - @Override - public String getTextRepresentation() { - return toString(); - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - } - - public enum DatabendBinaryArithmeticOperator implements Operator { - ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"); - - private String textRepr; - - DatabendBinaryArithmeticOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - public enum DatabendBinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); - //LIKE("LIKE"), NOT_LIKE("NOT LIKE"); - - private String textRepr; - - DatabendBinaryComparisonOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - public NewFunctionNode generateArgsForAggregate( - DatabendAggregateFunction aggregateFunction) { - return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); - } - - public Node generateAggregate() { - DatabendAggregateFunction aggrFunc = DatabendAggregateFunction.getRandom(); - return generateArgsForAggregate(aggrFunc); - } - - @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, DatabendUnaryPrefixOperator.NOT); - } - - @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, DatabendUnaryPostfixOperator.IS_NULL); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index ade2840e4..340e494ef 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -1,16 +1,17 @@ package sqlancer.databend.gen; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; - -import java.util.List; -import java.util.stream.Collectors; +import sqlancer.databend.DatabendToStringVisitor; public class DatabendInsertGenerator extends AbstractInsertGenerator { @@ -31,7 +32,7 @@ private SQLQueryAdapter generate() { List columns = table.getRandomNonEmptyColumnSubset(); sb.append(table.getName()); sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); sb.append(")"); sb.append(" VALUES "); insertColumns(columns); @@ -42,14 +43,13 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(DatabendColumn column) { // TODO: 等Databend实现DEFAULT关键字,暂时注入普通的value -// if (Randomly.getBooleanWithRatherLowProbability()) { -// sb.append("DEFAULT"); -// } else { -// sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); -// } - String value = DatabendToStringVisitor.asString( - new DatabendNewExpressionGenerator(globalState). - generateConstant(column.getType().getPrimitiveDataType(),column.isNullable())); //生成一个与column相同的常量类型 + // if (Randomly.getBooleanWithRatherLowProbability()) { + // sb.append("DEFAULT"); + // } else { + // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); + // } + String value = DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 9c953cb5e..f565a9be7 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -1,5 +1,10 @@ package sqlancer.databend.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.common.ast.newast.NewFunctionNode; @@ -9,19 +14,22 @@ import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; -import sqlancer.databend.ast.*; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; +import sqlancer.databend.ast.DatabendBinaryComparisonOperation; +import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; +import sqlancer.databend.ast.DatabendBinaryLogicalOperation; +import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendLikeOperation; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; -import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; -import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; -import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -public class DatabendNewExpressionGenerator extends - TypedExpressionGenerator, DatabendColumn, DatabendDataType> { +public class DatabendNewExpressionGenerator + extends TypedExpressionGenerator, DatabendColumn, DatabendDataType> { private final DatabendGlobalState globalState; private boolean allowAggregateFunctions; @@ -30,11 +38,11 @@ public DatabendNewExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; } + @Override public Node generateLeafNode(DatabendDataType dataType) { return generateConstant(dataType); } - @Override protected Node generateExpression(DatabendDataType type, int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { @@ -42,16 +50,16 @@ protected Node generateExpression(DatabendDataType type, int } switch (type) { - case BOOLEAN: - return generateBooleanExpression(depth); - case INT: - return generateIntExpression(depth); - case FLOAT: - case VARCHAR: - case NULL: - return generateConstant(type); - default: - throw new AssertionError(); + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case FLOAT: + case VARCHAR: + case NULL: + return generateConstant(type); + default: + throw new AssertionError(); } } @@ -63,50 +71,57 @@ public List> generateExpressions(int nr, DatabendDataTy return expressions; } - private enum IntExpression{ + private enum IntExpression { UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION } private Node generateIntExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } IntExpression intExpression = Randomly.fromOptions(IntExpression.values()); switch (intExpression) { - case UNARY_OPERATION: - return new DatabendUnaryPrefixOperation(generateExpression(DatabendDataType.INT,depth+1), - Randomly.getBoolean()? DatabendUnaryPrefixOperator.UNARY_PLUS : DatabendUnaryPrefixOperator.UNARY_MINUS); - case BINARY_ARITHMETIC_OPERATION: - return new DatabendBinaryArithmeticOperation(generateExpression(DatabendDataType.INT,depth+1), - generateExpression(DatabendDataType.INT,depth+1), - Randomly.fromOptions(DatabendBinaryArithmeticOperator.values())); - default: - throw new AssertionError(); + case UNARY_OPERATION: + return new DatabendUnaryPrefixOperation(generateExpression(DatabendDataType.INT, depth + 1), + Randomly.getBoolean() ? DatabendUnaryPrefixOperator.UNARY_PLUS + : DatabendUnaryPrefixOperator.UNARY_MINUS); + case BINARY_ARITHMETIC_OPERATION: + return new DatabendBinaryArithmeticOperation(generateExpression(DatabendDataType.INT, depth + 1), + generateExpression(DatabendDataType.INT, depth + 1), + Randomly.fromOptions(DatabendBinaryArithmeticOperator.values())); + default: + throw new AssertionError(); } } private enum BooleanExpression { POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; -// SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; + // SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; } Node generateBooleanExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); BooleanExpression option = Randomly.fromList(validOptions); switch (option) { - case POSTFIX_OPERATOR: - getPostfix(depth + 1); - case NOT: - getNOT(depth + 1); - case BETWEEN: //TODO (NULL BETWEEN NULL AND NULL) 返回的是 NULL 需要注意 - return getBetween(depth + 1); - case IN_OPERATION: - return getIn(depth + 1); - case BINARY_LOGICAL_OPERATOR: - return getBinaryLogical(depth + 1,DatabendDataType.BOOLEAN); - case BINARY_COMPARISON: - return getComparison(depth + 1); - case LIKE: - return getLike(depth + 1,DatabendDataType.VARCHAR); - default: - throw new AssertionError(); + case POSTFIX_OPERATOR: + getPostfix(depth + 1); + case NOT: + getNOT(depth + 1); + case BETWEEN: // TODO (NULL BETWEEN NULL AND NULL) 返回的是 NULL 需要注意 + return getBetween(depth + 1); + case IN_OPERATION: + return getIn(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return getBinaryLogical(depth + 1, DatabendDataType.BOOLEAN); + case BINARY_COMPARISON: + return getComparison(depth + 1); + case LIKE: + return getLike(depth + 1, DatabendDataType.VARCHAR); + default: + throw new AssertionError(); } } @@ -114,63 +129,58 @@ Node generateBooleanExpression(int depth) { Node getPostfix(int depth) { DatabendUnaryPostfixOperator randomOp = DatabendUnaryPostfixOperator.getRandom(); return new DatabendUnaryPostfixOperation( - generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), - randomOp,Randomly.getBoolean()); + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp, + Randomly.getBoolean()); } Node getNOT(int depth) { DatabendUnaryPrefixOperator op = DatabendUnaryPrefixOperator.NOT; - return new DatabendUnaryPrefixOperation( - generateExpression(Randomly.fromOptions(op.getInputDataTypes()), depth), - op,Randomly.getBoolean()); + return new DatabendUnaryPrefixOperation(generateExpression(op.getRandomInputDataTypes(), depth), op); } - Node getBetween(int depth){ - //跳过boolean + Node getBetween(int depth) { + // 跳过boolean DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); - return new NewBetweenOperatorNode(generateExpression(dataType,depth), - generateExpression(dataType,depth), generateExpression(dataType,depth), - Randomly.getBoolean()); + return new NewBetweenOperatorNode(generateExpression(dataType, depth), + generateExpression(dataType, depth), generateExpression(dataType, depth), Randomly.getBoolean()); } Node getIn(int depth) { DatabendDataType dataType = Randomly.fromOptions(DatabendDataType.values()); - Node leftExpr = generateExpression(dataType,depth); + Node leftExpr = generateExpression(dataType, depth); List> rightExprs = new ArrayList<>(); int nr = Randomly.smallNumber() + 1; - for(int i = 0; i < nr; i++) { - rightExprs.add(generateExpression(dataType,depth)); + for (int i = 0; i < nr; i++) { + rightExprs.add(generateExpression(dataType, depth)); } - return new NewInOperatorNode(leftExpr,rightExprs, Randomly.getBoolean()); + return new NewInOperatorNode(leftExpr, rightExprs, Randomly.getBoolean()); } - Node getBinaryLogical(int depth, DatabendDataType dataType){ - Node expr = generateExpression(dataType,depth); + Node getBinaryLogical(int depth, DatabendDataType dataType) { + Node expr = generateExpression(dataType, depth); int nr = Randomly.smallNumber() + 1; for (int i = 0; i < nr; i++) { - expr = new DatabendBinaryLogicalOperation(expr, - generateExpression(DatabendDataType.BOOLEAN, depth), + expr = new DatabendBinaryLogicalOperation(expr, generateExpression(DatabendDataType.BOOLEAN, depth), DatabendBinaryLogicalOperator.getRandom()); } return expr; } Node getComparison(int depth) { - //跳过boolean + // 跳过boolean DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); - Node leftExpr = generateExpression(dataType,depth); - Node rightExpr = generateExpression(dataType,depth); - DatabendBinaryComparisonOperation op = new DatabendBinaryComparisonOperation(leftExpr,rightExpr, + Node leftExpr = generateExpression(dataType, depth); + Node rightExpr = generateExpression(dataType, depth); + return new DatabendBinaryComparisonOperation(leftExpr, rightExpr, Randomly.fromOptions(DatabendBinaryComparisonOperator.values())); - return op; } Node getLike(int depth, DatabendDataType dataType) { - return new DatabendLikeOperation(generateExpression(dataType,depth) - ,generateExpression(dataType,depth), DatabendLikeOperation.DatabendLikeOperator.LIKE_OPERATOR); + return new DatabendLikeOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + DatabendLikeOperation.DatabendLikeOperator.LIKE_OPERATOR); } @Override @@ -180,16 +190,16 @@ public Node generatePredicate() { @Override public Node negatePredicate(Node predicate) { - return new DatabendUnaryPrefixOperation(predicate,DatabendUnaryPrefixOperator.NOT); + return new DatabendUnaryPrefixOperation(predicate, DatabendUnaryPrefixOperator.NOT); } @Override public Node isNull(Node predicate) { - return new DatabendUnaryPostfixOperation(predicate,DatabendUnaryPostfixOperator.IS_NULL); + return new DatabendUnaryPostfixOperation(predicate, DatabendUnaryPostfixOperator.IS_NULL); } - public Node generateConstant(DatabendDataType type,boolean isNullable) { - if(isNullable && Randomly.getBooleanWithSmallProbability()) { + public Node generateConstant(DatabendDataType type, boolean isNullable) { + if (isNullable && Randomly.getBooleanWithSmallProbability()) { createConstant(DatabendDataType.NULL); } return createConstant(type); @@ -206,20 +216,20 @@ public Node generateConstant(DatabendDataType type) { public Node createConstant(DatabendDataType type) { Randomly r = globalState.getRandomly(); switch (type) { - case INT: - //TODO 已支持数值型string转化，待添加 - return DatabendConstant.createIntConstant(r.getInteger()); - case BOOLEAN: - //TODO 已支持boolean型string转化，待添加 - return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); - case FLOAT: - return DatabendConstant.createFloatConstant((float) r.getDouble()); - case VARCHAR: - return DatabendConstant.createStringConstant(r.getString()); - case NULL: - return DatabendConstant.createNullConstant(); - default: - throw new AssertionError(type); + case INT: + // TODO 已支持数值型string转化但仍然不支持运算符计算，待添加 + return DatabendConstant.createIntConstant(r.getInteger()); + case BOOLEAN: + // TODO 已支持boolean型string转化但仍然不支持运算符计算，待添加 + return DatabendConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + return DatabendConstant.createFloatConstant((float) r.getDouble()); + case VARCHAR: + return DatabendConstant.createStringConstant(r.getString()); + case NULL: + return DatabendConstant.createNullConstant(); + default: + throw new AssertionError(type); } } @@ -239,21 +249,16 @@ protected boolean canGenerateColumnOfType(DatabendDataType type) { } public enum DatabendAggregateFunction { - MAX(1), - MIN(1), - AVG(1,DatabendDataType.INT,DatabendDataType.FLOAT), - COUNT(1), - SUM(1,DatabendDataType.INT,DatabendDataType.FLOAT), - STDDEV_POP(1), - COVAR_POP(1), COVAR_SAMP(2); - //, STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) + MAX(1), MIN(1), AVG(1, DatabendDataType.INT, DatabendDataType.FLOAT), COUNT(1), + SUM(1, DatabendDataType.INT, DatabendDataType.FLOAT), STDDEV_POP(1), COVAR_POP(1), COVAR_SAMP(2); + // , STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) private int nrArgs; private DatabendDataType[] dataTypes; - DatabendAggregateFunction(int nrArgs, DatabendDataType ...dataTypes) { + DatabendAggregateFunction(int nrArgs, DatabendDataType... dataTypes) { this.nrArgs = nrArgs; - this.dataTypes = dataTypes; + this.dataTypes = dataTypes.clone(); } public static DatabendAggregateFunction getRandom() { @@ -261,7 +266,7 @@ public static DatabendAggregateFunction getRandom() { } public DatabendDataType getRandomType() { - if(dataTypes.length == 0) { + if (dataTypes.length == 0) { return Randomly.fromOptions(DatabendDataType.values()); } else { return Randomly.fromOptions(dataTypes); @@ -277,7 +282,7 @@ public int getNrArgs() { public NewFunctionNode generateArgsForAggregate( DatabendAggregateFunction aggregateFunction) { return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs(),aggregateFunction.getRandomType()), + generateExpressions(aggregateFunction.getNrArgs(), aggregateFunction.getRandomType()), aggregateFunction); } @@ -287,9 +292,9 @@ public Node generateAggregate() { } public Node generateHavingClause() { - this.allowAggregateFunctions = true; + allowAggregateFunctions = true; Node expression = generateExpression(DatabendDataType.BOOLEAN); - this.allowAggregateFunctions = false; + allowAggregateFunctions = false; return expression; } diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 9ac3499b3..099c0aa9e 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -1,20 +1,20 @@ package sqlancer.databend.gen; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.DatabendSchema.DatabendTables; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; public final class DatabendRandomQuerySynthesizer { @@ -23,8 +23,8 @@ private DatabendRandomQuerySynthesizer() { public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); -// DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) -// .setColumns(targetTables.getColumns()); + // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) + // .setColumns(targetTables.getColumns()); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); @@ -58,16 +58,17 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int } if (Randomly.getBoolean()) { - select.setLimitClause(DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + select.setLimitClause( + DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } if (Randomly.getBoolean()) { select.setOffsetClause( DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } // TODO 待添加HavingClause -// if (Randomly.getBoolean()) { -// select.setHavingClause(gen.generateHavingClause()); -// } + // if (Randomly.getBoolean()) { + // select.setHavingClause(gen.generateHavingClause()); + // } return select; } diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index ba08f8acc..c83cd93e9 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -1,19 +1,19 @@ package sqlancer.databend.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; - -import java.util.ArrayList; -import java.util.List; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; public class DatabendTableGenerator { @@ -25,8 +25,8 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(tableName); sb.append("("); List columns = getNewColumns(); - TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = - new DatabendNewExpressionGenerator(globalState).setColumns(columns); + TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = new DatabendNewExpressionGenerator( + globalState).setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); @@ -34,42 +34,42 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType()); -// if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() -// && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { -// sb.append(" COLLATE "); -// sb.append(getRandomCollate()); -// } -// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { -// sb.append(" UNIQUE"); -// } + // if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + // && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { + // sb.append(" COLLATE "); + // sb.append(getRandomCollate()); + // } + // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + // sb.append(" UNIQUE"); + // } if (globalState.getDbmsSpecificOptions().testNotNullConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); } else { - sb.append(" NULL"); //Databend 默认字段为非空，这个将它默认设置为允许空 + sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 } -// if (globalState.getDbmsSpecificOptions().testCheckConstraints //databend 无check约束 -// && Randomly.getBooleanWithRatherLowProbability()) { -// sb.append(" CHECK("); -// sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); -// DatabendErrors.addExpressionErrors(errors); -// sb.append(")"); -// } + // if (globalState.getDbmsSpecificOptions().testCheckConstraints //databend 无check约束 + // && Randomly.getBooleanWithRatherLowProbability()) { + // sb.append(" CHECK("); + // sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); + // DatabendErrors.addExpressionErrors(errors); + // sb.append(")"); + // } if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); - sb.append(DatabendToStringVisitor.asString( //常量类型于字段类型等同 + sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); sb.append(")"); } } - //databend并没有索引 -// if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { -// errors.add("Invalid type for index"); -// List primaryKeyColumns = Randomly.nonEmptySubset(columns); -// sb.append(", PRIMARY KEY("); -// sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); -// sb.append(")"); -// } + // databend并没有索引 + // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { + // errors.add("Invalid type for index"); + // List primaryKeyColumns = Randomly.nonEmptySubset(columns); + // sb.append(", PRIMARY KEY("); + // sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + // sb.append(")"); + // } sb.append(")"); return new SQLQueryAdapter(sb.toString(), errors, true); } @@ -82,7 +82,7 @@ private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { DatabendCompositeDataType columnType = DatabendCompositeDataType.getRandomWithoutNull(); - String columnName = String.format("c%d%s", i,columnType.getPrimitiveDataType().toString()); + String columnName = String.format("c%d%s", i, columnType.getPrimitiveDataType().toString()); columns.add(new DatabendColumn(columnName, columnType, false, false)); } return columns; diff --git a/src/sqlancer/databend/gen/DatabendUpdateGenerator.java b/src/sqlancer/databend/gen/DatabendUpdateGenerator.java deleted file mode 100644 index e2f396c4b..000000000 --- a/src/sqlancer/databend/gen/DatabendUpdateGenerator.java +++ /dev/null @@ -1,48 +0,0 @@ -package sqlancer.databend.gen; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendTable; - -import java.util.List; - -public final class DatabendUpdateGenerator { - - private DatabendUpdateGenerator() { - } - - public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); - DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); - DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState).setColumns(table.getColumns()); - sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - Node expr; - if (Randomly.getBooleanWithSmallProbability()) { - expr = gen.generateExpression(); - DatabendErrors.addExpressionErrors(errors); - } else { - expr = gen.generateConstant(); - } - sb.append(DatabendToStringVisitor.asString(expr)); - } - DatabendErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); - } - -} diff --git a/src/sqlancer/databend/gen/DatabendViewGenerator.java b/src/sqlancer/databend/gen/DatabendViewGenerator.java index 55a7c6366..6daff1694 100644 --- a/src/sqlancer/databend/gen/DatabendViewGenerator.java +++ b/src/sqlancer/databend/gen/DatabendViewGenerator.java @@ -1,12 +1,11 @@ package sqlancer.databend.gen; -import sqlancer.GlobalState; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendToStringVisitor; public final class DatabendViewGenerator { @@ -27,7 +26,8 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { sb.append(i); } sb.append(") AS "); - sb.append(DatabendToStringVisitor.asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + sb.append(DatabendToStringVisitor + .asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); DatabendErrors.addGroupByErrors(errors); diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 451940a6e..7d315bbee 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -1,5 +1,12 @@ package sqlancer.databend.test; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -12,24 +19,20 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendCastOperation; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.gen.DatabendExpressionGenerator; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.*; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - public class DatabendNoRECOracle extends NoRECBase implements TestOracle { private final DatabendSchema s; @@ -42,20 +45,20 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { @Override public void check() throws SQLException { - DatabendTables randomTables = s.getRandomTableNonEmptyTables(); //随机获得nr张表 + DatabendTables randomTables = s.getRandomTableNonEmptyTables(); // 随机获得nr张表 List columns = randomTables.getColumns(); -// DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); + // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); //生成随机where条件，形式为ast + Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件，形式为ast -// System.out.println(DatabendToStringVisitor.asString(randomWhereCondition)); + // System.out.println(DatabendToStringVisitor.asString(randomWhereCondition)); List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); List> joins = DatabendJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); //禁用优化 + int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); // 禁用优化 int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { @@ -68,7 +71,7 @@ public void check() throws SQLException { } private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + List> joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); // DatabendExpression isTrue = DatabendPostfixOperation.create(randomWhereCondition, @@ -84,7 +87,7 @@ private int getSecondQuery(List> tableList, Node> tableList, Node> tableList, - List columns, Node randomWhereCondition, List> joins) - throws SQLException { + List columns, Node randomWhereCondition, + List> joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); // DatabendAggregate aggr = new DatabendAggregate( List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + .map((c) -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); // DatabendAggregateFunction.COUNT); // select.setFetchColumns(Arrays.asList(aggr)); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions(new DatabendExpressionGenerator(state).setColumns(columns).generateOrderBys()); + select.setOrderByExpressions( + new DatabendNewExpressionGenerator(state).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); int firstCount = 0; @@ -125,6 +130,7 @@ private int getFirstQueryCount(SQLConnection con, List> if (options.logEachSelect()) { logger.writeCurrent(optimizedQueryString); } + // System.out.println("First: " + optimizedQueryString); try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { while (rs.next()) { firstCount++; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index 9ce498234..f177b2921 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -18,17 +18,18 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; +import sqlancer.databend.ast.DatabendCastOperation; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.DatabendSchema.DatabendDataType; -import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; import sqlancer.databend.gen.DatabendNewExpressionGenerator.DatabendAggregateFunction; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendBinaryArithmeticOperator; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendCastOperation; -import sqlancer.databend.gen.DatabendExpressionGenerator.DatabendUnaryPrefixOperator; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; + public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { private String firstResult; @@ -55,9 +56,9 @@ public void check() throws SQLException { fetchColumns.add(gen.generateAggregate()); } select.setFetchColumns(Arrays.asList(aggregate)); -// if (Randomly.getBooleanWithRatherLowProbability()) { -// select.setOrderByExpressions(gen.generateOrderBys()); -// } + // if (Randomly.getBooleanWithRatherLowProbability()) { + // select.setOrderByExpressions(gen.generateOrderBys()); + // } originalQuery = DatabendToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); @@ -77,7 +78,8 @@ public void check() throws SQLException { } private String createMetamorphicUnionQuery(DatabendSelect select, - NewFunctionNode aggregate, List> from) { + NewFunctionNode aggregate, + List> from) { String metamorphicQuery; Node whereClause = gen.generateExpression(DatabendDataType.BOOLEAN); Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, @@ -109,7 +111,7 @@ private String getAggregateResult(String queryString) throws SQLException { try { resultString = result.getString(1); } catch (Exception e) { - System.out.println("Invalid integer format for value"); //TODO 超过integer范围无法格式化异常，还未有解决方案 + System.out.println("Invalid integer format for value"); // TODO 超过integer范围无法格式化异常，还未有解决方案 } } return resultString; @@ -122,7 +124,8 @@ private String getAggregateResult(String queryString) throws SQLException { } } - private List> mapped(NewFunctionNode aggregate) { + private List> mapped( + NewFunctionNode aggregate) { DatabendCastOperation count; switch (aggregate.getFunc()) { case COUNT: @@ -131,22 +134,23 @@ private List> mapped(NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), - DatabendAggregateFunction.SUM); - count = new DatabendCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), + NewFunctionNode sum = new NewFunctionNode<>( + aggregate.getArgs(), DatabendAggregateFunction.SUM); + count = new DatabendCastOperation( + new NewFunctionNode<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); return aliasArgs(Arrays.asList(sum, count)); case STDDEV_POP: NewFunctionNode sumSquared = new NewFunctionNode<>( Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), - DatabendBinaryArithmeticOperator.MULT)), + DatabendBinaryArithmeticOperator.MULTIPLICATION)), DatabendAggregateFunction.SUM); count = new DatabendCastOperation( new NewFunctionNode(aggregate.getArgs(), DatabendAggregateFunction.COUNT), new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); - NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), - DatabendAggregateFunction.AVG); + NewFunctionNode avg = new NewFunctionNode<>( + aggregate.getArgs(), DatabendAggregateFunction.AVG); return aliasArgs(Arrays.asList(sumSquared, count, avg)); default: throw new AssertionError(aggregate.getFunc()); @@ -183,7 +187,7 @@ private DatabendSelect getSelect(List> aggregates, List leftSelect.setWhereClause(whereClause); leftSelect.setJoinList(joinList); if (Randomly.getBooleanWithSmallProbability()) { -// leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); leftSelect.setGroupByExpressions(select.getFetchColumns()); } return leftSelect; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java index c0d090649..6ef132572 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -1,5 +1,11 @@ package sqlancer.databend.test; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; @@ -8,24 +14,18 @@ import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendSchema; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.ast.DatabendJoin; -import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; - -public class DatabendQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { +public class DatabendQueryPartitioningBase extends + TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { DatabendSchema s; DatabendTables targetTables; @@ -65,7 +65,7 @@ public void check() throws SQLException { List> generateFetchColumns() { List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { //TODO 为什么会返回 false 或 true 字段 + if (Randomly.getBoolean()) { // TODO 为什么会返回 false 或 true 字段 columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); } else { columns = generateRandomColumns(); diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java index 23572b110..8eae1e502 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java @@ -1,14 +1,14 @@ package sqlancer.databend.test; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; +import sqlancer.databend.DatabendToStringVisitor; public class DatabendQueryPartitioningDistinctTester extends DatabendQueryPartitioningBase { diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java index 0dbdda637..dd25b6d7f 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java @@ -1,19 +1,19 @@ package sqlancer.databend.test; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; -import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendExpression; public class DatabendQueryPartitioningGroupByTester extends DatabendQueryPartitioningBase { diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java index 18c17b452..72ad0795b 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -1,21 +1,21 @@ package sqlancer.databend.test; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Random; public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase implements TestOracle { @@ -30,19 +30,20 @@ public void check() throws SQLException { if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } -// boolean orderBy = Randomly.getBoolean(); - boolean orderBy = false; //关闭order by - if (orderBy) { //TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select -// select.setOrderByExpressions(gen.generateOrderBys()); + // boolean orderBy = Randomly.getBoolean(); + boolean orderBy = false; // 关闭order by + if (orderBy) { // TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select + // select.setOrderByExpressions(gen.generateOrderBys()); List> constants = new ArrayList<>(); - constants.add(new DatabendConstant.DatabendIntConstant(new Random().nextInt(select.getFetchColumns().size()) + 1)); + constants.add(new DatabendConstant.DatabendIntConstant( + Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByExpressions(constants); } -// select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + // select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); -// System.out.println(originalQueryString); + // System.out.println(originalQueryString); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setHavingClause(predicate); @@ -65,7 +66,7 @@ protected Node generatePredicate() { @Override List> generateFetchColumns() { - return Arrays.asList(gen.generateHavingClause()); + return Collections.singletonList(gen.generateHavingClause()); } } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java index 74f9a22c1..2b43d0ba0 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java @@ -1,15 +1,14 @@ package sqlancer.databend.test; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; - import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import sqlancer.ComparatorHelper; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendToStringVisitor; + public class DatabendQueryPartitioningWhereTester extends DatabendQueryPartitioningBase { public DatabendQueryPartitioningWhereTester(DatabendGlobalState state) { @@ -25,10 +24,11 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); - } + // boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + boolean orderBy = false; + // if (orderBy) { //TODO 待开启 + // select.setOrderByExpressions(gen.generateOrderBys()); + // } select.setWhereClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/test/sqlancer/databend/TestDatabendConnection.java b/test/sqlancer/databend/TestDatabendConnection.java index 91e0a1d53..db915140f 100644 --- a/test/sqlancer/databend/TestDatabendConnection.java +++ b/test/sqlancer/databend/TestDatabendConnection.java @@ -1,6 +1,5 @@ package sqlancer.databend; -import com.mongodb.annotations.ThreadSafe; import org.junit.jupiter.api.Test; import sqlancer.Main; import sqlancer.Randomly; @@ -10,11 +9,12 @@ public class TestDatabendConnection { @Test - void testConnection(){ + void testConnection() { assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix","databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC),"--host","192.168.81.134","--port","3307","--username","user1" - ,"databend","--oracle","NoREC"})); + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + "--host", "192.168.191.151", "--port", "3307", "--username", "user1", "--password", "1234", + "databend", "--oracle", "HAVING" })); } } diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 2cf53eef2..bc7e0ea1c 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -69,4 +69,4 @@ public void testClickHouseTLPAggregate() { "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPAggregate")); } -} +} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDB.java index d36d8610f..da981faca 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDB.java @@ -17,4 +17,4 @@ public void testDuckDB() { "QUERY_PARTITIONING" })); } -} +} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestMySQLTLP.java b/test/sqlancer/dbms/TestMySQLTLP.java index 3d3e7d107..793d0f807 100644 --- a/test/sqlancer/dbms/TestMySQLTLP.java +++ b/test/sqlancer/dbms/TestMySQLTLP.java @@ -21,4 +21,4 @@ public void testMySQL() { "mysql", "--oracle", "TLP_WHERE" })); } -} +} \ No newline at end of file From 4daedd927af1e72fc2c80d21f7b3e99a01799e9f Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 11 Sep 2022 16:39:40 +0800 Subject: [PATCH 183/856] feat(databend): add ci test Signed-off-by: Yisong Han --- .github/workflows/main.yml | 29 ++++- databases/database0.db | Bin 0 -> 12288 bytes databases/databaseconnectiontest.db | 0 ...DatabendQueryPartitioningHavingTester.java | 1 - test/sqlancer/dbms/TestDatabend.java | 106 ++++++++++++++++++ 5 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 databases/database0.db create mode 100644 databases/databaseconnectiontest.db create mode 100644 test/sqlancer/dbms/TestDatabend.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e11a8d053..12e0d4bee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -117,6 +117,33 @@ jobs: - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test + databend: + name: DBMS Tests (Databend) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up Databend + run: | + sudo apt update + sudo apt install mysql-client + LASTEST_TAG=$(curl -s GET https://api.github.com/repos/datafuselabs/databend/tags\?per_page\=1 | jq -r '.[].name') + curl -LJO https://github.com/datafuselabs/databend/releases/download/${LASTEST_TAG}/databend-${LASTEST_TAG}-x86_64-unknown-linux-musl.tar.gz + mkdir ./databend && tar xzvf databend-${LASTEST_TAG}-x86_64-unknown-linux-musl.tar.gz -C ./databend + ./databend/bin/databend-query & + - name: Create SQLancer user + run: mysql -uroot -h127.0.0.1 -P3307 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" + - name: Run Tests + run: | + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test + duckdb: name: DBMS Tests (DuckDB) runs-on: ubuntu-latest @@ -320,4 +347,4 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb \ No newline at end of file diff --git a/databases/database0.db b/databases/database0.db new file mode 100644 index 0000000000000000000000000000000000000000..2072f0a2b682817091df8ed8176e371cfef499dc GIT binary patch literal 12288 zcmeI2&ubGw6vw}t-89;?tR8akAZ($U2HKJ~KTK{WZPh^9(j>LswAus;TB!uvYel?y z_dn@Lyz5cWgWyGP;z>{t1;1}6)7`X!*Me`C*_nAW@6CIkH?xqi?X~81zZ>)(K6NvF4}pWco0_S$80&QT*-z%)nd9GjxGb&fV;W4UL$ ziPXhR=FIaN9&_C;_E_h`GVJY5dborQT9t9Tu3fdwu_ns`Jq#E}BlJ~bG7{sx#JPbR zoF{Qb5lMuzjAV||lTmbzG$eLASK_}M>(8+-F(0=tYg!$#AD?06c}k;I;?l@k8AbY6 zV^RA6Z7Yg%a=VkO%R2%pM3BlZhmt#ggLY8A_r zi03vw*=b@+?rd0u_-dGcy8~N{x<%W|p}jgaI^?$$Mj*OT*0S8h$K#hs>}rxvySw5n zXUJ)uUR79^_$;#&|EVRshB?Xp$W!|YtrYky@+p}R+XdQ`j1*`$VkS1K=pD23Hy_1z zmDa>kA&j#cwkotPYnfcj{+3-HkJDt@oHUbj%X^ROihPB{J4a+Bue^IsY9fgmIRr{-w~bg?8?cq*)=E6>=k*BTxV%ZJsffD%vw wN Date: Sun, 11 Sep 2022 18:19:31 +0800 Subject: [PATCH 184/856] fix(databend): code adjustment Signed-off-by: Yisong Han --- .github/workflows/main.yml | 2 +- .gitignore | 4 + .settings/org.eclipse.jdt.core.prefs | 9 +- databases/database0.db | Bin 12288 -> 0 bytes databases/databaseconnectiontest.db | 0 dependency-reduced-pom.xml | 433 ------------------ src/sqlancer/GlobalState.java | 9 +- src/sqlancer/databend/DatabendErrors.java | 2 + src/sqlancer/databend/DatabendProvider.java | 24 +- src/sqlancer/databend/DatabendSchema.java | 13 +- .../DatabendBinaryComparisonOperation.java | 6 +- .../ast/DatabendBinaryLogicalOperation.java | 40 +- src/sqlancer/databend/ast/DatabendJoin.java | 1 - .../gen/DatabendNewExpressionGenerator.java | 4 +- .../gen/DatabendRandomQuerySynthesizer.java | 15 +- .../databend/gen/DatabendViewGenerator.java | 19 +- .../databend/test/DatabendNoRECOracle.java | 5 - ...abendQueryPartitioningAggregateTester.java | 12 +- .../databend/TestDatabendConnection.java | 20 - test/sqlancer/dbms/TestClickHouse.java | 2 +- test/sqlancer/dbms/TestDatabend.java | 123 ++--- test/sqlancer/dbms/TestDuckDB.java | 2 +- test/sqlancer/dbms/TestMySQLTLP.java | 2 +- 23 files changed, 108 insertions(+), 639 deletions(-) delete mode 100644 databases/database0.db delete mode 100644 databases/databaseconnectiontest.db delete mode 100644 dependency-reduced-pom.xml delete mode 100644 test/sqlancer/databend/TestDatabendConnection.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 12e0d4bee..bfbbe60cc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -347,4 +347,4 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb \ No newline at end of file + run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb diff --git a/.gitignore b/.gitignore index dcec833f6..48efcf83a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,8 +2,12 @@ target/ .classpath .settings/org.eclipse.core.resources.prefs .settings/org.eclipse.m2e.core.prefs +.settings/org.eclipse.jdt.core.prefs .project .checkstyle *.DS_Store .idea SQLancer.iml +dependency-reduced-pom.xml +database0.db +databaseconnectiontest.db diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 3af1c5ad8..0a659c1a7 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=11 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,7 +28,6 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore -org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -74,7 +73,6 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore -org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -112,6 +110,5 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=11 +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/databases/database0.db b/databases/database0.db deleted file mode 100644 index 2072f0a2b682817091df8ed8176e371cfef499dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12288 zcmeI2&ubGw6vw}t-89;?tR8akAZ($U2HKJ~KTK{WZPh^9(j>LswAus;TB!uvYel?y z_dn@Lyz5cWgWyGP;z>{t1;1}6)7`X!*Me`C*_nAW@6CIkH?xqi?X~81zZ>)(K6NvF4}pWco0_S$80&QT*-z%)nd9GjxGb&fV;W4UL$ ziPXhR=FIaN9&_C;_E_h`GVJY5dborQT9t9Tu3fdwu_ns`Jq#E}BlJ~bG7{sx#JPbR zoF{Qb5lMuzjAV||lTmbzG$eLASK_}M>(8+-F(0=tYg!$#AD?06c}k;I;?l@k8AbY6 zV^RA6Z7Yg%a=VkO%R2%pM3BlZhmt#ggLY8A_r zi03vw*=b@+?rd0u_-dGcy8~N{x<%W|p}jgaI^?$$Mj*OT*0S8h$K#hs>}rxvySw5n zXUJ)uUR79^_$;#&|EVRshB?Xp$W!|YtrYky@+p}R+XdQ`j1*`$VkS1K=pD23Hy_1z zmDa>kA&j#cwkotPYnfcj{+3-HkJDt@oHUbj%X^ROihPB{J4a+Bue^IsY9fgmIRr{-w~bg?8?cq*)=E6>=k*BTxV%ZJsffD%vw wN - - 4.0.0 - com.sqlancer - sqlancer - SQLancer - 2.0.0 - SQLancer finds logic bugs in Database Management Systems through automatic testing - http://www.sqlancer.com/ - - - mrigger - Manuel Rigger - manuel.rigger@inf.ethz.ch - https://www.manuelrigger.at/ - ETH Zurich - https://ethz.ch/ - - architect - developer - - - - - - MIT License - https://github.com/sqlancer/sqlancer/blob/master/LICENSE.md - repo - - - - scm:git:git://github.com/sqlancer/sqlancer.git - scm:git:ssh://github.com:sqlancer/sqlancer.git - https://github.com/sqlancer/sqlancer/ - - - src - test - - - maven-shade-plugin - 3.3.0 - - - package - - shade - - - - - com.beust:jcommander - - - - - - - - maven-antrun-plugin - 3.1.0 - - - package - - run - - - - - - - - - - - - - maven-surefire-plugin - 2.22.2 - - - org.jacoco - jacoco-maven-plugin - 0.8.8 - - - - prepare-agent - - - - report - test - - report - - - - - - maven-compiler-plugin - 3.10.1 - - - org.codehaus.plexus - plexus-compiler-eclipse - 2.12.0 - - - org.eclipse.jdt - ecj - 3.28.0 - - - org.codehaus.plexus - plexus-compiler-api - 2.12.0 - - - - 11 - 11 - - ${project.basedir}/.settings/org.eclipse.jdt.core.prefs - - eclipse - true - true - - - - net.revelc.code.formatter - formatter-maven-plugin - 2.19.0 - - - eclipseformat - verify - - validate - - - - - - maven-dependency-plugin - 3.2.0 - - - copy-dependencies - prepare-package - - copy-dependencies - - - ${project.build.directory}/lib - false - false - true - jcommander - - - - - - maven-jar-plugin - 3.2.2 - - true - - - true - lib/ - sqlancer.Main - - - - - - org.codehaus.mojo - xml-maven-plugin - 1.0.2 - - - xml-check-format - verify - - check-format - - - - - - maven-checkstyle-plugin - 3.1.2 - - - checkstyle - verify - - check - - - true - - - - - - com.puppycrawl.tools - checkstyle - 10.3 - - - - configs/checkstyle.xml - - - - maven-pmd-plugin - 3.14.0 - - - pmd - verify - - check - - - 2 - true - - configs/pmd-rules.xml - - - - - - - com.github.spotbugs - spotbugs-maven-plugin - 4.7.0.0 - - - spotbugs - verify - - check - - - configs/spotbugs-exclude.xml - High - - - - - - - - - jdk-8-config - - ${java.home}/../bin/javadoc - - - - jdk-11-config - - ${java.home}/bin/javadoc - - - - release-steps - - - - maven-source-plugin - 3.2.1 - - - attach-sources - - jar-no-fork - - - - - - maven-javadoc-plugin - 3.4.0 - - - attach-javadocs - - jar - - - - - 8 - - - - maven-gpg-plugin - 3.0.1 - - - sign-artifacts - verify - - sign - - - - --pinentry-mode - loopback - - - - - - - - - - - - com.google.auto.service - auto-service - 1.0.1 - compile - - - org.postgresql - postgresql - 42.4.0 - compile - - - org.xerial - sqlite-jdbc - 3.36.0.3 - compile - - - mysql - mysql-connector-java - 8.0.29 - compile - - - org.mariadb.jdbc - mariadb-java-client - 3.0.5 - compile - - - org.duckdb - duckdb_jdbc - 0.4.0 - compile - - - org.junit.jupiter - junit-jupiter-engine - 5.8.2 - test - - - junit-platform-engine - org.junit.platform - - - junit-jupiter-api - org.junit.jupiter - - - apiguardian-api - org.apiguardian - - - - - org.slf4j - slf4j-simple - 1.7.36 - compile - - - ru.yandex.clickhouse - clickhouse-jdbc - 0.3.2 - compile - - - com.h2database - h2 - 2.1.214 - compile - - - org.mongodb - mongodb-driver-sync - 4.1.1 - compile - - - com.arangodb - arangodb-java-driver - 6.9.0 - compile - - - - - - maven-jxr-plugin - 3.2.0 - - - - - - ossrh - Central Repository OSSRH - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - UTF-8 - - diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 9c48dd900..64b5c731e 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -1,7 +1,5 @@ package sqlancer; -import java.sql.SQLException; - import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractSchema; @@ -144,12 +142,7 @@ protected void setSchema(S schema) { } public void updateSchema() throws Exception { - try { - setSchema(readSchema()); - } catch (SQLException sqlException) { - System.out.printf("%s readSchema SQLException%n", databaseName); - sqlException.printStackTrace(); - } + setSchema(readSchema()); for (AbstractTable table : schema.getDatabaseTables()) { table.recomputeCount(); } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 92c7b17f7..60ea7ca9c 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -51,6 +51,8 @@ public static void addExpressionErrors(ExpectedErrors errors) { // removed column errors.add("Contents of view were altered: types don't match!"); errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + // TODO Databend待修复的bug（union schema error mismatch） + errors.add("unexpected end of file (failed to fill whole buffer)"); } private static void addRegexErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 793650798..9a94241d0 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -1,6 +1,5 @@ package sqlancer.databend; -import java.io.File; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -74,7 +73,7 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); // case DELETE: // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); - // case CREATE_VIEW: + // case CREATE_VIEW: //TODO 暂时关闭create view // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); @@ -111,22 +110,6 @@ public void generateDatabase(DatabendGlobalState globalState) throws Exception { se.executeStatements(); // 在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 } - public void tryDeleteFile(String fname) { - try { - File f = new File(fname); - f.delete(); - } catch (Exception e) { - } - } - - public void tryDeleteDatabase(String dbpath) { - if (dbpath.equals("") || dbpath.equals(":memory:")) { - return; - } - tryDeleteFile(dbpath); - tryDeleteFile(dbpath + ".wal"); - } - @Override public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLException { String username = globalState.getOptions().getUserName(); @@ -156,11 +139,6 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE globalState.getState().logStatement("USE " + databaseName); } - // try (Statement s = con.createStatement()) { - // s.execute("set enable_planner_v2 = 0;"); - // globalState.getState().logStatement("set enable_planner_v2 = 0;"); - // } - return new SQLConnection(con); } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 643f53dd9..98d7acafd 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -255,21 +255,14 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { List tableNames = null; tableNames = new ArrayList<>(); - // SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema != 'system' and table_schema != - // 'INFORMATION_SCHEMA' and table_type='BASE TABLE' - // "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ",databaseName + final String sqlStatement = String.format( "SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE table_schema = '%s' and table_type='BASE TABLE' ", databaseName); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(sqlStatement)) { - try { // 没有catch的话rs.next()会报SQLException：Not a navigable ResultSet - while (rs.next()) { - tableNames.add(rs.getString("table_name")); - } - } catch (Exception e) { - // e.printStackTrace(); - System.out.println("TableNames->SQLException：Not a navigable ResultSet"); + while (rs.next()) { + tableNames.add(rs.getString("table_name")); } } } diff --git a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java index 9e8e43a51..cf63ad17e 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java @@ -12,10 +12,8 @@ public DatabendBinaryComparisonOperation(Node left, Node"), GREATER_EQUALS(">="); + EQUALS("="), IS_DISTINCT("IS DISTINCT FROM"), IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), NOT_EQUALS("!="), + LESS("<"), LESS_EQUALS("<="), GREATER(">"), GREATER_EQUALS(">="); private final String textRepresentation; diff --git a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java index 14ff984e6..355ec88a5 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java @@ -13,43 +13,7 @@ public DatabendBinaryLogicalOperation(Node left, Node> getJoins( TableReferenceNode rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); - // DatabendExpressionGenerator joinGen = new DatabendExpressionGenerator(globalState).setColumns(columns); DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState) .setColumns(columns); diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index f565a9be7..43d38939f 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -107,9 +107,9 @@ Node generateBooleanExpression(int depth) { BooleanExpression option = Randomly.fromList(validOptions); switch (option) { case POSTFIX_OPERATOR: - getPostfix(depth + 1); + return getPostfix(depth + 1); case NOT: - getNOT(depth + 1); + return getNOT(depth + 1); case BETWEEN: // TODO (NULL BETWEEN NULL AND NULL) 返回的是 NULL 需要注意 return getBetween(depth + 1); case IN_OPERATION: diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 099c0aa9e..286764c6e 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -23,8 +23,6 @@ private DatabendRandomQuerySynthesizer() { public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); - // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(globalState) - // .setColumns(targetTables.getColumns()); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState) .setColumns(targetTables.getColumns()); DatabendSelect select = new DatabendSelect(); @@ -50,12 +48,13 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); - } - if (Randomly.getBoolean()) { - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - } + // if (Randomly.getBoolean()) {//TODO order by超过实际行数 + // select.setOrderByExpressions(gen.generateOrderBys()); + // } + + // if (Randomly.getBoolean()) { //TODO group by超过实际行数 + // select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + // } if (Randomly.getBoolean()) { select.setLimitClause( diff --git a/src/sqlancer/databend/gen/DatabendViewGenerator.java b/src/sqlancer/databend/gen/DatabendViewGenerator.java index 6daff1694..55cc6ffc5 100644 --- a/src/sqlancer/databend/gen/DatabendViewGenerator.java +++ b/src/sqlancer/databend/gen/DatabendViewGenerator.java @@ -17,15 +17,16 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { StringBuilder sb = new StringBuilder("CREATE "); sb.append("VIEW "); sb.append(globalState.getSchema().getFreeViewName()); - sb.append("("); - for (int i = 0; i < nrColumns; i++) { - if (i != 0) { - sb.append(", "); - } - sb.append("c"); - sb.append(i); - } - sb.append(") AS "); + // sb.append("("); + // for (int i = 0; i < nrColumns; i++) { + // if (i != 0) { + // sb.append(", "); + // } + // sb.append("c"); + // sb.append(i); + // } + // sb.append(") AS "); + sb.append(" AS "); sb.append(DatabendToStringVisitor .asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 7d315bbee..cb27fa78b 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -47,13 +47,10 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { public void check() throws SQLException { DatabendTables randomTables = s.getRandomTableNonEmptyTables(); // 随机获得nr张表 List columns = randomTables.getColumns(); - // DatabendExpressionGenerator gen = new DatabendExpressionGenerator(state).setColumns(columns); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件，形式为ast - // System.out.println(DatabendToStringVisitor.asString(randomWhereCondition)); - List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -87,7 +84,6 @@ private int getSecondQuery(List> tableList, Node> if (options.logEachSelect()) { logger.writeCurrent(optimizedQueryString); } - // System.out.println("First: " + optimizedQueryString); try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { while (rs.next()) { firstCount++; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index f177b2921..59d6695ca 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -111,7 +111,7 @@ private String getAggregateResult(String queryString) throws SQLException { try { resultString = result.getString(1); } catch (Exception e) { - System.out.println("Invalid integer format for value"); // TODO 超过integer范围无法格式化异常，还未有解决方案 + throw new IgnoreMeException(); // TODO 超过integer范围无法格式化异常，还未有解决方案 } } return resultString; @@ -171,7 +171,7 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List leftSelect.setFromList(from); leftSelect.setWhereClause(whereClause); leftSelect.setJoinList(joinList); - if (Randomly.getBooleanWithSmallProbability()) { - leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - leftSelect.setGroupByExpressions(select.getFetchColumns()); - } + // if (Randomly.getBooleanWithSmallProbability()) { + // leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); //TODO group by超过实际行数 + // leftSelect.setGroupByExpressions(select.getFetchColumns());// TODO group by不能放入聚合函数 + // } return leftSelect; } diff --git a/test/sqlancer/databend/TestDatabendConnection.java b/test/sqlancer/databend/TestDatabendConnection.java deleted file mode 100644 index db915140f..000000000 --- a/test/sqlancer/databend/TestDatabendConnection.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.databend; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; -import sqlancer.Randomly; -import sqlancer.dbms.TestConfig; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class TestDatabendConnection { - @Test - void testConnection() { - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "192.168.191.151", "--port", "3307", "--username", "user1", "--password", "1234", - "databend", "--oracle", "HAVING" })); - } -} diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index bc7e0ea1c..2cf53eef2 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -69,4 +69,4 @@ public void testClickHouseTLPAggregate() { "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPAggregate")); } -} \ No newline at end of file +} diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index 94a86aef0..a9575fd7d 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -21,66 +21,7 @@ public void testDatabendNoREC() { "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); } - @Test - public void testDatabendTLPWhere() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "WHERE")); - } - - @Test - public void testDatabendTLPGroupBy() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "GROUP_BY")); - } - - @Test - public void testDatabendTLPHaving() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "HAVING")); - } - - @Test - public void testDatabendTLPDistinct() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "DISTINCT")); - } - - @Test - public void testDatabendTLPAggregate() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "AGGREGATE")); - } - + // TODO Databend待修复的bug（union schema error mismatch）https://github.com/datafuselabs/databend/issues/7463 @Test public void testDatabendTLPQueryPartitioning() { String databendAvailable = System.getenv("DATABEND_AVAILABLE"); @@ -93,13 +34,73 @@ public void testDatabendTLPQueryPartitioning() { "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); } + // @Test + // public void testDatabendTLPWhere() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "WHERE")); + // } + // + // @Test + // public void testDatabendTLPGroupBy() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "GROUP_BY")); + // } + // + // @Test + // public void testDatabendTLPHaving() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "HAVING")); + // } + // + // @Test + // public void testDatabendTLPDistinct() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "DISTINCT")); + // } + // + // @Test + // public void testDatabendTLPAggregate() { + // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + // assumeTrue(databendIsAvailable); + // assertEquals(0, + // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "AGGREGATE")); + // } + // @Test // void testConnection() { // assertEquals(0, // Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, // "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - // "--host", "192.168.191.151", "--port", "3307", "--username", "user1", "--password", "1234", + // "--host", "127.0.0.1", "--port", "3307", "--username", "user1", "--password", "1234", // "databend", "--oracle", "HAVING" })); // } diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDB.java index da981faca..d36d8610f 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDB.java @@ -17,4 +17,4 @@ public void testDuckDB() { "QUERY_PARTITIONING" })); } -} \ No newline at end of file +} diff --git a/test/sqlancer/dbms/TestMySQLTLP.java b/test/sqlancer/dbms/TestMySQLTLP.java index 793d0f807..3d3e7d107 100644 --- a/test/sqlancer/dbms/TestMySQLTLP.java +++ b/test/sqlancer/dbms/TestMySQLTLP.java @@ -21,4 +21,4 @@ public void testMySQL() { "mysql", "--oracle", "TLP_WHERE" })); } -} \ No newline at end of file +} From 26b893a09adf43eeae5e1f7e6d9e92d3efd1a3cc Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Tue, 4 Oct 2022 13:29:50 +0400 Subject: [PATCH 185/856] Added wrapped exceptions support --- src/sqlancer/common/query/SQLQueryAdapter.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index c2e6706cc..14dbd523e 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -96,9 +96,19 @@ public String getUnterminatedQueryString() { } public void checkException(Exception e) throws AssertionError { - if (!expectedErrors.errorIsExpected(e.getMessage())) { - throw new AssertionError(query, e); + Throwable ex = e; + + while (true) { + if (expectedErrors.errorIsExpected(ex.getMessage())) { + return; + } else if (ex.getCause() != null) { + ex = ex.getCause(); + } else { + break; + } } + + throw new AssertionError(query, ex); } @Override From 2265a9a989256db64bfb9b382d8e0f8367904806 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Tue, 4 Oct 2022 15:39:26 +0400 Subject: [PATCH 186/856] Reworked checkExceptions method --- src/sqlancer/common/query/SQLQueryAdapter.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 14dbd523e..01fcc8539 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -98,17 +98,15 @@ public String getUnterminatedQueryString() { public void checkException(Exception e) throws AssertionError { Throwable ex = e; - while (true) { + while (ex != null) { if (expectedErrors.errorIsExpected(ex.getMessage())) { return; - } else if (ex.getCause() != null) { - ex = ex.getCause(); } else { - break; + ex = ex.getCause(); } } - throw new AssertionError(query, ex); + throw new AssertionError(query, e); } @Override From 46b7eecfdfabe71ab10431d981e0fcdc4654929d Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Tue, 4 Oct 2022 17:37:30 +0400 Subject: [PATCH 187/856] Added Yugabyte API support --- configs/spotbugs-exclude.xml | 7 +- pom.xml | 10 + src/sqlancer/yugabyte/ycql/YCQLErrors.java | 21 + src/sqlancer/yugabyte/ycql/YCQLOptions.java | 47 ++ src/sqlancer/yugabyte/ycql/YCQLProvider.java | 156 +++++ src/sqlancer/yugabyte/ycql/YCQLSchema.java | 267 ++++++++ .../yugabyte/ycql/YCQLToStringVisitor.java | 64 ++ .../yugabyte/ycql/ast/YCQLConstant.java | 173 +++++ .../yugabyte/ycql/ast/YCQLExpression.java | 8 + .../yugabyte/ycql/ast/YCQLSelect.java | 18 + .../ycql/gen/YCQLAlterTableGenerator.java | 48 ++ .../ycql/gen/YCQLDeleteGenerator.java | 31 + .../ycql/gen/YCQLExpressionGenerator.java | 297 +++++++++ .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 56 ++ .../ycql/gen/YCQLInsertGenerator.java | 65 ++ .../ycql/gen/YCQLRandomQuerySynthesizer.java | 55 ++ .../yugabyte/ycql/gen/YCQLTableGenerator.java | 57 ++ .../ycql/gen/YCQLUpdateGenerator.java | 56 ++ .../yugabyte/ycql/test/YCQLFuzzer.java | 73 +++ .../yugabyte/ysql/YSQLCompoundDataType.java | 45 ++ .../ysql/YSQLExpectedValueVisitor.java | 152 +++++ .../yugabyte/ysql/YSQLGlobalState.java | 127 ++++ src/sqlancer/yugabyte/ysql/YSQLOptions.java | 98 +++ src/sqlancer/yugabyte/ysql/YSQLProvider.java | 366 +++++++++++ src/sqlancer/yugabyte/ysql/YSQLSchema.java | 329 ++++++++++ .../yugabyte/ysql/YSQLToStringVisitor.java | 329 ++++++++++ src/sqlancer/yugabyte/ysql/YSQLVisitor.java | 120 ++++ .../yugabyte/ysql/ast/YSQLAggregate.java | 58 ++ src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java | 35 + .../ysql/ast/YSQLBetweenOperation.java | 63 ++ .../ast/YSQLBinaryArithmeticOperation.java | 106 +++ .../ysql/ast/YSQLBinaryBitOperation.java | 46 ++ .../ast/YSQLBinaryComparisonOperation.java | 135 ++++ .../ysql/ast/YSQLBinaryLogicalOperation.java | 88 +++ .../ysql/ast/YSQLBinaryRangeOperation.java | 71 ++ .../yugabyte/ysql/ast/YSQLCastOperation.java | 45 ++ .../yugabyte/ysql/ast/YSQLCollate.java | 33 + .../yugabyte/ysql/ast/YSQLColumnValue.java | 34 + .../ysql/ast/YSQLConcatOperation.java | 37 ++ .../yugabyte/ysql/ast/YSQLConstant.java | 611 ++++++++++++++++++ .../yugabyte/ysql/ast/YSQLExpression.java | 14 + .../yugabyte/ysql/ast/YSQLFunction.java | 283 ++++++++ .../ast/YSQLFunctionWithUnknownResult.java | 174 +++++ .../yugabyte/ysql/ast/YSQLInOperation.java | 65 ++ src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java | 49 ++ .../yugabyte/ysql/ast/YSQLOrderByTerm.java | 42 ++ .../ysql/ast/YSQLPOSIXRegularExpression.java | 65 ++ .../ysql/ast/YSQLPostfixOperation.java | 146 +++++ .../yugabyte/ysql/ast/YSQLPostfixText.java | 36 ++ .../ysql/ast/YSQLPrefixOperation.java | 115 ++++ .../yugabyte/ysql/ast/YSQLSelect.java | 135 ++++ .../yugabyte/ysql/ast/YSQLSimilarTo.java | 39 ++ .../ysql/gen/YSQLAlterTableGenerator.java | 182 ++++++ .../ysql/gen/YSQLAnalyzeGenerator.java | 37 ++ .../ysql/gen/YSQLClusterGenerator.java | 32 + .../ysql/gen/YSQLCommentGenerator.java | 68 ++ .../yugabyte/ysql/gen/YSQLCommon.java | 441 +++++++++++++ .../ysql/gen/YSQLDeleteGenerator.java | 46 ++ .../ysql/gen/YSQLDiscardGenerator.java | 39 ++ .../ysql/gen/YSQLDropIndexGenerator.java | 41 ++ .../ysql/gen/YSQLExpressionGenerator.java | 563 ++++++++++++++++ .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 153 +++++ .../ysql/gen/YSQLInsertGenerator.java | 127 ++++ .../ysql/gen/YSQLNotifyGenerator.java | 45 ++ .../ysql/gen/YSQLRandomQueryGenerator.java | 62 ++ .../ysql/gen/YSQLReindexGenerator.java | 58 ++ .../ysql/gen/YSQLSequenceGenerator.java | 90 +++ .../yugabyte/ysql/gen/YSQLSetGenerator.java | 196 ++++++ .../ysql/gen/YSQLStatisticsGenerator.java | 74 +++ .../yugabyte/ysql/gen/YSQLTableGenerator.java | 247 +++++++ .../ysql/gen/YSQLTableGroupGenerator.java | 24 + .../ysql/gen/YSQLTransactionGenerator.java | 27 + .../ysql/gen/YSQLTruncateGenerator.java | 41 ++ .../ysql/gen/YSQLUpdateGenerator.java | 76 +++ .../ysql/gen/YSQLVacuumGenerator.java | 19 + .../yugabyte/ysql/gen/YSQLViewGenerator.java | 73 +++ .../yugabyte/ysql/oracle/YSQLCatalog.java | 94 +++ .../yugabyte/ysql/oracle/YSQLFuzzer.java | 108 ++++ .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 165 +++++ .../YSQLPivotedQuerySynthesisOracle.java | 147 +++++ .../oracle/tlp/YSQLTLPAggregateOracle.java | 194 ++++++ .../yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 117 ++++ .../ysql/oracle/tlp/YSQLTLPHavingOracle.java | 66 ++ .../ysql/oracle/tlp/YSQLTLPWhereOracle.java | 45 ++ 84 files changed, 9296 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/yugabyte/ycql/YCQLErrors.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLOptions.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLProvider.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLSchema.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java create mode 100644 src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLGlobalState.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLOptions.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLProvider.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLSchema.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLVisitor.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java create mode 100644 src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java create mode 100644 src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java diff --git a/configs/spotbugs-exclude.xml b/configs/spotbugs-exclude.xml index 366e8ac55..5cc1baf73 100644 --- a/configs/spotbugs-exclude.xml +++ b/configs/spotbugs-exclude.xml @@ -11,5 +11,10 @@ - + + + + + + diff --git a/pom.xml b/pom.xml index 234d805f1..fdb287a10 100644 --- a/pom.xml +++ b/pom.xml @@ -286,6 +286,16 @@ postgresql 42.5.0 + + com.ing.data + cassandra-jdbc-wrapper + 4.6.0 + + + com.yugabyte + jdbc-yugabytedb + 42.3.0 + org.xerial sqlite-jdbc diff --git a/src/sqlancer/yugabyte/ycql/YCQLErrors.java b/src/sqlancer/yugabyte/ycql/YCQLErrors.java new file mode 100644 index 000000000..2bcddb71c --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLErrors.java @@ -0,0 +1,21 @@ +package sqlancer.yugabyte.ycql; + +import sqlancer.common.query.ExpectedErrors; + +public final class YCQLErrors { + + private YCQLErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.add("Signature mismatch in call to builtin function"); + errors.add("Qualified name not allowed for column reference"); + errors.add("Datatype Mismatch"); + errors.add("Invalid Datatype"); + errors.add("Invalid CQL Statement"); + errors.add("Invalid SQL Statement"); + errors.add("Order by clause contains invalid expression"); + errors.add("Invalid Function Call"); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLOptions.java b/src/sqlancer/yugabyte/ycql/YCQLOptions.java new file mode 100644 index 000000000..5955d28dd --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLOptions.java @@ -0,0 +1,47 @@ +package sqlancer.yugabyte.ycql; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ycql.YCQLOptions.YCQLOracleFactory; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.test.YCQLFuzzer; + +@Parameters(commandDescription = "YCQL") +public class YCQLOptions implements DBMSSpecificOptions { + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates = 5; + + @Parameter(names = "--datacenter", description = "YCQL datacenter, can be found in system.local table", arity = 1) + public String datacenter = "datacenter1"; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(YCQLOracleFactory.FUZZER); + + public enum YCQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YCQLGlobalState globalState) throws SQLException { + return new YCQLFuzzer(globalState); + } + + } + } + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLProvider.java b/src/sqlancer/yugabyte/ycql/YCQLProvider.java new file mode 100644 index 000000000..3d408f8c1 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLProvider.java @@ -0,0 +1,156 @@ +package sqlancer.yugabyte.ycql; + +import static sqlancer.yugabyte.ycql.YCQLSchema.getTableNames; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.gen.YCQLAlterTableGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLDeleteGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLIndexGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLInsertGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLRandomQuerySynthesizer; +import sqlancer.yugabyte.ycql.gen.YCQLTableGenerator; +import sqlancer.yugabyte.ycql.gen.YCQLUpdateGenerator; + +@AutoService(DatabaseProvider.class) +public class YCQLProvider extends SQLProviderAdapter { + + public YCQLProvider() { + super(YCQLGlobalState.class, YCQLOptions.class); + } + + public enum Action implements AbstractAction { + + ALTER(YCQLAlterTableGenerator::getQuery), // + INSERT(YCQLInsertGenerator::getQuery), // + CREATE_INDEX(YCQLIndexGenerator::getQuery), // + DELETE(YCQLDeleteGenerator::generate), // + UPDATE(YCQLUpdateGenerator::getQuery), // + EXPLAIN((g) -> { + ExpectedErrors errors = new ExpectedErrors(); + YCQLErrors.addExpressionErrors(errors); + return new SQLQueryAdapter( + "EXPLAIN " + YCQLToStringVisitor + .asString(YCQLRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), + errors); + }); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(YCQLGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(YCQLGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case ALTER: + return r.getInteger(0, 10); + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case CREATE_INDEX: + case UPDATE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + case EXPLAIN: + return r.getInteger(0, 2); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + default: + throw new AssertionError(a); + } + } + + public static class YCQLGlobalState extends SQLGlobalState { + + @Override + protected YCQLSchema readSchema() throws SQLException { + return YCQLSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(YCQLGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new YCQLTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + YCQLProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLException { + try { + Class.forName("com.ing.data.cassandra.jdbc.CassandraDriver"); + } catch (ClassNotFoundException e) { + e.printStackTrace(); + } + final String host = globalState.getOptions().getHost(); + final String url = "jdbc:cassandra://%s:9042/%s?localdatacenter=%s"; + final Connection connection = DriverManager.getConnection( + String.format(url, host, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); + + try (Statement stmt = connection.createStatement()) { + try { + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); + } catch (Exception se) { + // try again + List tableNames = getTableNames( + new SQLConnection(DriverManager.getConnection(String.format(url, host, + globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))), + globalState.getDatabaseName()); + for (String tableName : tableNames) { + stmt.execute("DROP TABLE " + globalState.getDatabaseName() + "." + tableName); + } + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); + } + + stmt.execute("CREATE KEYSPACE IF NOT EXISTS " + globalState.getDatabaseName()); + } + + return new SQLConnection(DriverManager.getConnection(String.format(url, host, globalState.getDatabaseName(), + globalState.getDbmsSpecificOptions().datacenter))); + } + + @Override + public String getDBMSName() { + return "ycql"; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java new file mode 100644 index 000000000..4845520d2 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -0,0 +1,267 @@ +package sqlancer.yugabyte.ycql; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; + +public class YCQLSchema extends AbstractSchema { + + public enum YCQLDataType { + + INT, VARCHAR, BOOLEAN, FLOAT, DATE, TIMESTAMP; + + public static YCQLDataType getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public static class YCQLCompositeDataType { + + private final YCQLDataType dataType; + + private final int size; + + public YCQLCompositeDataType(YCQLDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public YCQLDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static YCQLCompositeDataType getRandom() { + YCQLDataType type = YCQLDataType.getRandom(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case BOOLEAN: + case VARCHAR: + case DATE: + case TIMESTAMP: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new YCQLCompositeDataType(type, size); + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 8: + return Randomly.fromOptions("BIGINT"); + case 4: + return Randomly.fromOptions("INTEGER", "INT"); + case 2: + return Randomly.fromOptions("SMALLINT"); + case 1: + return Randomly.fromOptions("TINYINT"); + default: + throw new AssertionError(size); + } + case VARCHAR: + return "VARCHAR"; + case FLOAT: + switch (size) { + case 8: + return Randomly.fromOptions("DOUBLE"); + case 4: + return Randomly.fromOptions("FLOAT"); + default: + throw new AssertionError(size); + } + case BOOLEAN: + return Randomly.fromOptions("BOOLEAN"); + case TIMESTAMP: + return Randomly.fromOptions("TIMESTAMP"); + case DATE: + return Randomly.fromOptions("DATE"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + } + + public static class YCQLColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + + public YCQLColumn(String name, YCQLCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class YCQLTables extends AbstractTables { + + public YCQLTables(List tables) { + super(tables); + } + + } + + public YCQLSchema(List databaseTables) { + super(databaseTables); + } + + public YCQLTables getRandomTableNonEmptyTables() { + return new YCQLTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + private static YCQLCompositeDataType getColumnType(String typeString) { + YCQLDataType primitiveType; + int size = -1; + switch (typeString.toUpperCase()) { + case "INT": + case "INTEGER": + primitiveType = YCQLDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = YCQLDataType.INT; + size = 2; + break; + case "BIGINT": + primitiveType = YCQLDataType.INT; + size = 8; + break; + case "TINYINT": + primitiveType = YCQLDataType.INT; + size = 1; + break; + case "VARCHAR": + case "TEXT": + primitiveType = YCQLDataType.VARCHAR; + break; + case "FLOAT": + primitiveType = YCQLDataType.FLOAT; + size = 4; + break; + case "DOUBLE": + primitiveType = YCQLDataType.FLOAT; + size = 8; + break; + case "BOOLEAN": + primitiveType = YCQLDataType.BOOLEAN; + break; + case "DATE": + primitiveType = YCQLDataType.DATE; + break; + case "TIMESTAMP": + primitiveType = YCQLDataType.TIMESTAMP; + break; + case "INTERVAL": + throw new IgnoreMeException(); + // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP + // '1969-12-29 06:50:27')) + default: + throw new IgnoreMeException(); + } + return new YCQLCompositeDataType(primitiveType, size); + } + + public static class YCQLTable extends AbstractRelationalTable { + + public YCQLTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static YCQLSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con, databaseName); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; + } + List databaseColumns = getTableColumns(con, databaseName, tableName); + boolean isView = tableName.startsWith("v"); + YCQLTable t = new YCQLTable(tableName, databaseColumns, isView); + for (YCQLColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new YCQLSchema(databaseTables); + } + + public static List getTableNames(SQLConnection con, String databaseName) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + String.format("select * from system_schema.tables where keyspace_name = '%s'", databaseName))) { + while (rs.next()) { + tableNames.add(rs.getString("table_name")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "select * from system_schema.columns where keyspace_name = '%s' and table_name = '%s'", + databaseName, tableName))) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("type"); + boolean isPrimaryKey = rs.getString("kind").contentEquals("partition_key"); + YCQLColumn c = new YCQLColumn(columnName, getColumnType(dataType), isPrimaryKey, true); + columns.add(c); + } + } + } + return columns; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java new file mode 100644 index 000000000..ddc7e6e8a --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java @@ -0,0 +1,64 @@ +package sqlancer.yugabyte.ycql; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.yugabyte.ycql.ast.YCQLConstant; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import sqlancer.yugabyte.ycql.ast.YCQLSelect; + +public class YCQLToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof YCQLConstant) { + visit((YCQLConstant) expr); + } else if (expr instanceof YCQLSelect) { + visit((YCQLSelect) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(YCQLConstant constant) { + sb.append(constant.toString()); + } + + private void visit(YCQLSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(Node expr) { + YCQLToStringVisitor visitor = new YCQLToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java new file mode 100644 index 000000000..dd85b4b66 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java @@ -0,0 +1,173 @@ +package sqlancer.yugabyte.ycql.ast; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +import sqlancer.common.ast.newast.Node; + +public class YCQLConstant implements Node { + + private YCQLConstant() { + } + + public static class YCQLNullConstant extends YCQLConstant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class YCQLIntConstant extends YCQLConstant { + + private final long value; + + public YCQLIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class YCQLDoubleConstant extends YCQLConstant { + + private final double value; + + public YCQLDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + } + + public static class YCQLTextConstant extends YCQLConstant { + + private final String value; + + public YCQLTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static class YCQLDateConstant extends YCQLConstant { + + public String textRepr; + + public YCQLDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("'%s'", textRepr); + } + + } + + public static class YCQLTimestampConstant extends YCQLConstant { + + public String textRepr; + + public YCQLTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("'%s'", textRepr); + } + + } + + public static class YCQLBooleanConstant extends YCQLConstant { + + private final boolean value; + + public YCQLBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static Node createStringConstant(String text) { + return new YCQLTextConstant(text); + } + + public static Node createFloatConstant(double val) { + return new YCQLDoubleConstant(val); + } + + public static Node createIntConstant(long val) { + return new YCQLIntConstant(val); + } + + public static Node createNullConstant() { + return new YCQLNullConstant(); + } + + public static Node createBooleanConstant(boolean val) { + return new YCQLBooleanConstant(val); + } + + public static Node createDateConstant(long integer) { + return new YCQLDateConstant(integer); + } + + public static Node createTimestampConstant(long integer) { + return new YCQLTimestampConstant(integer); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java b/src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java new file mode 100644 index 000000000..4177344eb --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLExpression.java @@ -0,0 +1,8 @@ +package sqlancer.yugabyte.ycql.ast; + +public interface YCQLExpression { + + default YCQLConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java new file mode 100644 index 000000000..646182875 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java @@ -0,0 +1,18 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class YCQLSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java new file mode 100644 index 000000000..d15d7fe1c --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLAlterTableGenerator.java @@ -0,0 +1,48 @@ +package sqlancer.yugabyte.ycql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLCompositeDataType; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; + +public final class YCQLAlterTableGenerator { + + private YCQLAlterTableGenerator() { + } + + enum Action { + ADD_COLUMN, DROP_COLUMN + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(YCQLCompositeDataType.getRandom().toString()); + break; + case DROP_COLUMN: + sb.append("DROP "); + sb.append(table.getRandomColumn().getName()); + break; + default: + throw new AssertionError(action); + } + + errors.add("Alter key column. Can't alter key column"); + errors.add("cannot remove a key column"); + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java new file mode 100644 index 000000000..108cd1be9 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java @@ -0,0 +1,31 @@ +package sqlancer.yugabyte.ycql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLErrors; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; + +public final class YCQLDeleteGenerator { + + private YCQLDeleteGenerator() { + } + + public static SQLQueryAdapter generate(YCQLGlobalState globalState) { + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(YCQLToStringVisitor.asString( + new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); + } + + YCQLErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java new file mode 100644 index 000000000..296be911d --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java @@ -0,0 +1,297 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLDataType; +import sqlancer.yugabyte.ycql.ast.YCQLConstant; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; + +public final class YCQLExpressionGenerator extends UntypedExpressionGenerator, YCQLColumn> { + + private final YCQLGlobalState globalState; + + public YCQLExpressionGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, FUNC, BETWEEN, IN + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBoolean()) { + YCQLAggregateFunction aggregate = YCQLAggregateFunction.getRandom(); + allowAggregates = false; + return new NewFunctionNode<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); + } + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case BINARY_COMPARISON: + Operator op = YCQLBinaryComparisonOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = YCQLBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new NewBinaryOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), YCQLBinaryArithmeticOperator.getRandom()); + case FUNC: + DBFunction func = DBFunction.getRandom(); + return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new NewBetweenOperatorNode(generateExpression(depth + 1), + generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new NewInOperatorNode(generateExpression(depth + 1), + generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); + default: + throw new AssertionError(expr); + } + } + + @Override + protected Node generateColumn() { + YCQLColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode(column); + } + + @Override + public Node generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return YCQLConstant.createNullConstant(); + } + YCQLDataType type = YCQLDataType.getRandom(); + switch (type) { + case INT: + return YCQLConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DATE: + return YCQLConstant.createDateConstant(globalState.getRandomly().getInteger()); + case TIMESTAMP: + return YCQLConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case VARCHAR: + return YCQLConstant.createStringConstant(globalState.getRandomly().getString()); + case BOOLEAN: + return YCQLConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + return YCQLConstant.createFloatConstant(globalState.getRandomly().getDouble()); + default: + throw new AssertionError(); + } + } + + @Override + public List> generateOrderBys() { + List> expr = super.generateOrderBys(); + List> newExpr = new ArrayList<>(expr.size()); + for (Node curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + }; + + public enum YCQLAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1), SUM(1); + + private final int nrArgs; + + YCQLAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static YCQLAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } + + public enum DBFunction { + // YCQL functions + BLOB(1), // + TIMEUUID(1), // + DATE(0), // + TIME(0), // + TIMESTAMP(0), // + BIGINT(1), // + UUID(0); // + // // extras + // PARTITION_HASH(2), // + // WRITETIME(1), // + // TTL(1); // + + private final int nrArgs; + private final boolean isVariadic; + + DBFunction(int nrArgs) { + this(nrArgs, false); + } + + DBFunction(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static DBFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + + } + + public enum YCQLUnaryPostfixOperator implements Operator { + + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private final String textRepr; + + YCQLUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static YCQLUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum YCQLUnaryPrefixOperator implements Operator { + + NOT("NOT"), PLUS("+"), MINUS("-"); + + private final String textRepr; + + YCQLUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static YCQLUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum YCQLBinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum YCQLBinaryArithmeticOperator implements Operator { + ADD("+"), SUB("-"), MULT("*"), DIV("/"); + + private String textRepr; + + YCQLBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum YCQLBinaryComparisonOperator implements Operator { + + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); + + private final String textRepr; + + YCQLBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public NewFunctionNode generateArgsForAggregate( + YCQLAggregateFunction aggregateFunction) { + return new NewFunctionNode( + generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + } + + public Node generateAggregate() { + YCQLAggregateFunction aggrFunc = YCQLAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, YCQLUnaryPrefixOperator.NOT); + } + + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, YCQLUnaryPostfixOperator.IS_NULL); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java new file mode 100644 index 000000000..2dc565196 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -0,0 +1,56 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; + +public final class YCQLIndexGenerator { + + private YCQLIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + if (Randomly.getBoolean()) { + errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + sb.append("UNIQUE "); + } + sb.append("INDEX "); + sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); + sb.append(" ON "); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append("("); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + } + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + Node expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(); + sb.append(YCQLToStringVisitor.asString(expr)); + } + errors.add("Query timed out after PT2S"); + errors.add("Invalid SQL Statement"); + errors.add("Invalid CQL Statement"); + errors.add( + "Invalid Table Definition. Transactions cannot be enabled in an index of a table without transactions enabled."); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java new file mode 100644 index 000000000..a27ec9c92 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -0,0 +1,65 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ycql.YCQLErrors; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; + +public class YCQLInsertGenerator extends AbstractInsertGenerator { + + private final YCQLGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public YCQLInsertGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + return new YCQLInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getColumns(); + sb.append(globalState.getDatabaseName()).append(".").append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + + errors.add("Invalid Arguments"); + errors.add("Null Argument for Primary Key"); + + YCQLErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertColumns(List columns) { + sb.append("("); + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + insertValue(columns.get(nrColumn)); + } + sb.append(")"); + } + + @Override + protected void insertValue(YCQLColumn tiDBColumn) { + // TODO: select a more meaningful value + sb.append(YCQLToStringVisitor.asString(new YCQLExpressionGenerator(globalState).generateConstant())); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java new file mode 100644 index 000000000..7a10009ad --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java @@ -0,0 +1,55 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTables; +import sqlancer.yugabyte.ycql.ast.YCQLConstant; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import sqlancer.yugabyte.ycql.ast.YCQLSelect; + +public final class YCQLRandomQuerySynthesizer { + + private YCQLRandomQuerySynthesizer() { + } + + public static YCQLSelect generateSelect(YCQLGlobalState globalState, int nrColumns) { + YCQLTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YCQLExpressionGenerator gen = new YCQLExpressionGenerator(globalState).setColumns(targetTables.getColumns()); + YCQLSelect select = new YCQLSelect(); + List> columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + Node expression = gen.generateExpression(); + columns.add(expression); + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + Optional> table = tables.stream() + .map(t -> new TableReferenceNode(t)).findFirst(); + select.setFromList(table.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(Randomly.nonEmptySubset(select.getFetchColumns())); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YCQLConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(YCQLConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + return select; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java new file mode 100644 index 000000000..148a1cdbc --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java @@ -0,0 +1,57 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLCompositeDataType; + +public class YCQLTableGenerator { + + public SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + sb.append(columns.get(i).getType()); + // todo PK, STATIC + } + errors.add("Query timed out after PT2S"); + errors.add("Invalid type for index"); + List primaryKeyColumns = Randomly.nonEmptySubset(columns); + sb.append(", PRIMARY KEY("); + sb.append(primaryKeyColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(")"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + YCQLCompositeDataType columnType = YCQLCompositeDataType.getRandom(); + columns.add(new YCQLColumn(columnName, columnType, false, false)); + } + return columns; + } + +} diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java new file mode 100644 index 000000000..d417ca96a --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -0,0 +1,56 @@ +package sqlancer.yugabyte.ycql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLErrors; +import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; +import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; +import sqlancer.yugabyte.ycql.ast.YCQLExpression; + +public final class YCQLUpdateGenerator { + + private YCQLUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { + StringBuilder sb = new StringBuilder("UPDATE "); + ExpectedErrors errors = new ExpectedErrors(); + YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + YCQLExpressionGenerator gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(" SET "); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append("="); + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + YCQLErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(YCQLToStringVisitor.asString(expr)); + } + + errors.add("Invalid Arguments"); + errors.add("Invalid CQL Statement"); + errors.add("Invalid SQL Statement"); + errors.add("Datatype Mismatch"); + errors.add("Null Argument for Primary Key"); + errors.add("Missing Argument for Primary Key"); + + YCQLErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java new file mode 100644 index 000000000..8dee32c3e --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java @@ -0,0 +1,73 @@ +package sqlancer.yugabyte.ycql.test; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ycql.YCQLProvider; +import sqlancer.yugabyte.ycql.YCQLToStringVisitor; +import sqlancer.yugabyte.ycql.gen.YCQLRandomQuerySynthesizer; + +public class YCQLFuzzer implements TestOracle { + private final YCQLProvider.YCQLGlobalState globalState; + private final List testQueries; + private final ExpectedErrors errors = new ExpectedErrors(); + + public YCQLFuzzer(YCQLProvider.YCQLGlobalState globalState) { + this.globalState = globalState; + + errors.add("Query timed out after PT2S"); + errors.add("Datatype Mismatch"); + errors.add("Invalid CQL Statement"); + errors.add("Invalid SQL Statement"); + errors.add("Invalid Arguments"); + errors.add("Invalid Function Call"); + + testQueries = new ArrayList<>(); + + testQueries.add(new SelectQuery()); + testQueries.add(new ActionQuery(YCQLProvider.Action.UPDATE)); + testQueries.add(new ActionQuery(YCQLProvider.Action.DELETE)); + testQueries.add(new ActionQuery(YCQLProvider.Action.INSERT)); + } + + @Override + public void check() throws Exception { + Query s = testQueries.get(globalState.getRandomly().getInteger(0, testQueries.size())); + globalState.executeStatement(s.getQuery(globalState, errors)); + globalState.getManager().incrementSelectQueryCount(); + } + + private static class Query { + public SQLQueryAdapter getQuery(YCQLProvider.YCQLGlobalState state, ExpectedErrors errors) throws Exception { + throw new IllegalAccessException("Should be implemented"); + }; + } + + private static class ActionQuery extends Query { + private final YCQLProvider.Action action; + + ActionQuery(YCQLProvider.Action action) { + this.action = action; + } + + @Override + public SQLQueryAdapter getQuery(YCQLProvider.YCQLGlobalState state, ExpectedErrors errors) throws Exception { + return action.getQuery(state); + } + } + + private static class SelectQuery extends Query { + + @Override + public SQLQueryAdapter getQuery(YCQLProvider.YCQLGlobalState state, ExpectedErrors errors) throws Exception { + return new SQLQueryAdapter( + YCQLToStringVisitor.asString( + YCQLRandomQuerySynthesizer.generateSelect(state, Randomly.smallNumber() + 1)) + ";", + errors); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java b/src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java new file mode 100644 index 000000000..3a8b8946d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLCompoundDataType.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql; + +import java.util.Optional; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public final class YSQLCompoundDataType { + + private final YSQLDataType dataType; + private final YSQLCompoundDataType elemType; + private final Integer size; + + private YSQLCompoundDataType(YSQLDataType dataType, YSQLCompoundDataType elemType, Integer size) { + this.dataType = dataType; + this.elemType = elemType; + this.size = size; + } + + public static YSQLCompoundDataType create(YSQLDataType type, int size) { + return new YSQLCompoundDataType(type, null, size); + } + + public static YSQLCompoundDataType create(YSQLDataType type) { + return new YSQLCompoundDataType(type, null, null); + } + + public YSQLDataType getDataType() { + return dataType; + } + + public YSQLCompoundDataType getElemType() { + if (elemType == null) { + throw new AssertionError(); + } + return elemType; + } + + public Optional getSize() { + if (size == null) { + return Optional.empty(); + } else { + return Optional.of(size); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java new file mode 100644 index 000000000..cf175ca39 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLExpectedValueVisitor.java @@ -0,0 +1,152 @@ +package sqlancer.yugabyte.ysql; + +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLSubquery; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; + +public final class YSQLExpectedValueVisitor implements YSQLVisitor { + + private static final int NR_TABS = 0; + private final StringBuilder sb = new StringBuilder(); + + private void print(YSQLExpression expr) { + YSQLToStringVisitor v = new YSQLToStringVisitor(); + v.visit(expr); + sb.append("\t".repeat(NR_TABS)); + sb.append(v.get()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + @Override + public void visit(YSQLConstant constant) { + print(constant); + } + + @Override + public void visit(YSQLPostfixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(YSQLColumnValue c) { + print(c); + } + + @Override + public void visit(YSQLPrefixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(YSQLSelect op) { + visit(op.getWhereClause()); + } + + @Override + public void visit(YSQLOrderByTerm op) { + + } + + @Override + public void visit(YSQLFunction f) { + print(f); + for (int i = 0; i < f.getArguments().length; i++) { + visit(f.getArguments()[i]); + } + } + + @Override + public void visit(YSQLCastOperation cast) { + print(cast); + visit(cast.getExpression()); + } + + @Override + public void visit(YSQLBetweenOperation op) { + print(op); + visit(op.getExpr()); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(YSQLInOperation op) { + print(op); + visit(op.getExpr()); + for (YSQLExpression right : op.getListElements()) { + visit(right); + } + } + + @Override + public void visit(YSQLPostfixText op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(YSQLAggregate op) { + print(op); + for (YSQLExpression expr : op.getArgs()) { + visit(expr); + } + } + + @Override + public void visit(YSQLSimilarTo op) { + print(op); + visit(op.getString()); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + } + + @Override + public void visit(YSQLPOSIXRegularExpression op) { + print(op); + visit(op.getString()); + visit(op.getRegex()); + } + + @Override + public void visit(YSQLFromTable from) { + print(from); + } + + @Override + public void visit(YSQLSubquery subquery) { + print(subquery); + } + + @Override + public void visit(YSQLBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public String get() { + return sb.toString(); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLGlobalState.java b/src/sqlancer/yugabyte/ysql/YSQLGlobalState.java new file mode 100644 index 000000000..093f06aed --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLGlobalState.java @@ -0,0 +1,127 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; + +public class YSQLGlobalState extends SQLGlobalState { + + public static final char IMMUTABLE = 'i'; + public static final char STABLE = 's'; + public static final char VOLATILE = 'v'; + // store and allow filtering by function volatility classifications + private final Map functionsAndTypes = new HashMap<>(); + private List operators = Collections.emptyList(); + private List collates = Collections.emptyList(); + private List opClasses = Collections.emptyList(); + private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + + @Override + public void setConnection(SQLConnection con) { + super.setConnection(con); + try { + this.opClasses = getOpclasses(getConnection()); + this.operators = getOperators(getConnection()); + this.collates = getCollnames(getConnection()); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + @Override + public YSQLSchema readSchema() throws SQLException { + return YSQLSchema.fromConnection(getConnection(), getDatabaseName()); + } + + private List getCollnames(SQLConnection con) throws SQLException { + List opClasses = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("SELECT collname FROM pg_collation WHERE collname LIKE '%utf8' or collname = 'C';")) { + while (rs.next()) { + opClasses.add(rs.getString(1)); + } + } + } + return opClasses; + } + + private List getOpclasses(SQLConnection con) throws SQLException { + List opClasses = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("select opcname FROM pg_opclass;")) { + while (rs.next()) { + opClasses.add(rs.getString(1)); + } + } + } + return opClasses; + } + + private List getOperators(SQLConnection con) throws SQLException { + List opClasses = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { + while (rs.next()) { + opClasses.add(rs.getString(1)); + } + } + } + return opClasses; + } + + public List getOperators() { + return operators; + } + + public String getRandomOperator() { + return Randomly.fromList(operators); + } + + public List getCollates() { + return collates; + } + + public String getRandomCollate() { + return Randomly.fromList(collates); + } + + public List getOpClasses() { + return opClasses; + } + + public String getRandomOpclass() { + return Randomly.fromList(opClasses); + } + + public void addFunctionAndType(String functionName, Character functionType) { + this.functionsAndTypes.put(functionName, functionType); + } + + public Map getFunctionsAndTypes() { + return this.functionsAndTypes; + } + + public void setDefaultAllowedFunctionTypes() { + this.allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + } + + public List getAllowedFunctionTypes() { + return this.allowedFunctionTypes; + } + + public void setAllowedFunctionTypes(List types) { + this.allowedFunctionTypes = types; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLOptions.java b/src/sqlancer/yugabyte/ysql/YSQLOptions.java new file mode 100644 index 000000000..4cd299660 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLOptions.java @@ -0,0 +1,98 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ysql.YSQLOptions.YSQLOracleFactory; +import sqlancer.yugabyte.ysql.oracle.YSQLCatalog; +import sqlancer.yugabyte.ysql.oracle.YSQLFuzzer; +import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; +import sqlancer.yugabyte.ysql.oracle.YSQLPivotedQuerySynthesisOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPAggregateOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPHavingOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPWhereOracle; + +@Parameters(separators = "=", commandDescription = "YSQL (default port: " + YSQLOptions.DEFAULT_PORT + + ", default host: " + YSQLOptions.DEFAULT_HOST) +public class YSQLOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 5433; + + @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) + public boolean allowBulkInsert; + + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for YSQL") + public List oracle = Arrays.asList(YSQLOracleFactory.QUERY_PARTITIONING); + + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) + public boolean testCollations = true; + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the YSQL server", arity = 1) + public String connectionURL = String.format("jdbc:yugabytedb://%s:%d/yugabyte", YSQLOptions.DEFAULT_HOST, + YSQLOptions.DEFAULT_PORT); + + @Override + public List getTestOracleFactory() { + return oracle; + } + + public enum YSQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLFuzzer(globalState); + } + }, + CATALOG { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLCatalog(globalState); + } + }, + NOREC { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLNoRECOracle(globalState); + } + }, + PQS { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + HAVING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + List oracles = new ArrayList<>(); + oracles.add(new YSQLTLPWhereOracle(globalState)); + oracles.add(new YSQLTLPHavingOracle(globalState)); + oracles.add(new YSQLTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java new file mode 100644 index 000000000..278e5316e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -0,0 +1,366 @@ +package sqlancer.yugabyte.ysql; + +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.YSQLOptions.YSQLOracleFactory; +import sqlancer.yugabyte.ysql.gen.YSQLAlterTableGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLAnalyzeGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLCommentGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLDeleteGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLDiscardGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLDropIndexGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLIndexGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLInsertGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLNotifyGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLSequenceGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLSetGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTableGroupGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTransactionGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLTruncateGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLUpdateGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLVacuumGenerator; +import sqlancer.yugabyte.ysql.gen.YSQLViewGenerator; + +@AutoService(DatabaseProvider.class) +public class YSQLProvider extends SQLProviderAdapter { + + public static final Object CREATION_LOCK = new Object(); + /** + * Generate only data types and expressions that are understood by PQS. + */ + public static boolean generateOnlyKnown; + protected String entryURL; + protected String username; + protected String password; + protected String entryPath; + protected String host; + protected int port; + protected String testURL; + protected String databaseName; + protected String createDatabaseCommand; + + public YSQLProvider() { + super(YSQLGlobalState.class, YSQLOptions.class); + } + + protected YSQLProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + public static int mapActions(YSQLGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + switch (a) { + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 3); + break; + case DISCARD: + case DROP_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case COMMIT: + nrPerformed = r.getInteger(0, 0); + break; + case ALTER_TABLE: + nrPerformed = r.getInteger(0, 5); + break; + case RESET: + nrPerformed = r.getInteger(0, 3); + break; + case ANALYZE: + nrPerformed = r.getInteger(0, 3); + break; + case TABLEGROUP: + nrPerformed = r.getInteger(0, 3); + break; + case DELETE: + case RESET_ROLE: + case VACUUM: + case SET_CONSTRAINTS: + case SET: + case COMMENT_ON: + case NOTIFY: + case LISTEN: + case UNLISTEN: + case CREATE_SEQUENCE: + case TRUNCATE: + nrPerformed = r.getInteger(0, 2); + break; + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + + } + + @Override + public void generateDatabase(YSQLGlobalState globalState) throws Exception { + readFunctions(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + prepareTables(globalState); + } + + @Override + public SQLConnection createDatabase(YSQLGlobalState globalState) throws SQLException { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == YSQLOracleFactory.PQS)) { + generateOnlyKnown = true; + } + + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + entryPath = "/yugabyte"; + entryURL = globalState.getDbmsSpecificOptions().connectionURL; + String entryDatabaseName = entryPath.substring(1); + databaseName = globalState.getDatabaseName(); + + try { + URI uri = new URI(entryURL); + String userInfoURI = uri.getUserInfo(); + String pathURI = uri.getPath(); + if (userInfoURI != null) { + // username and password specified in URL take precedence + if (userInfoURI.contains(":")) { + String[] userInfo = userInfoURI.split(":", 2); + username = userInfo[0]; + password = userInfo[1]; + } else { + username = userInfoURI; + password = null; + } + int userInfoIndex = entryURL.indexOf(userInfoURI); + String preUserInfo = entryURL.substring(0, userInfoIndex); + String postUserInfo = entryURL.substring(userInfoIndex + userInfoURI.length() + 1); + entryURL = preUserInfo + postUserInfo; + } + if (pathURI != null) { + entryPath = pathURI; + } + if (host == null) { + host = uri.getHost(); + } + if (port == MainOptions.NO_SET_PORT) { + port = uri.getPort(); + } + entryURL = String.format("jdbc:yugabytedb://%s:%d/%s", host, port, entryDatabaseName); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + + createDatabaseSync(globalState, entryDatabaseName); + + int databaseIndex = entryURL.indexOf("/" + entryDatabaseName) + 1; + String preDatabaseName = entryURL.substring(0, databaseIndex); + String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); + testURL = preDatabaseName + databaseName + postDatabaseName; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + + return new SQLConnection(createConnectionSafely(testURL, username, password)); + } + + @Override + public String getDBMSName() { + return "ysql"; + } + + // for some reason yugabyte unable to create few databases simultaneously + private void createDatabaseSync(YSQLGlobalState globalState, String entryDatabaseName) throws SQLException { + synchronized (CREATION_LOCK) { + exceptionLessSleep(5000); + + Connection con = createConnectionSafely(entryURL, username, password); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + createDatabaseCommand = getCreateDatabaseCommand(globalState); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + } + } + + private Connection createConnectionSafely(String entryURL, String user, String password) { + Connection con = null; + IllegalStateException lastException = new IllegalStateException("Empty exception"); + long endTime = System.currentTimeMillis() + 30000; + while (System.currentTimeMillis() < endTime) { + try { + con = DriverManager.getConnection(entryURL, user, password); + break; + } catch (SQLException throwables) { + lastException = new IllegalStateException(throwables); + } + } + + if (con == null) { + throw lastException; + } + + return con; + } + + protected void readFunctions(YSQLGlobalState globalState) throws SQLException { + SQLQueryAdapter query = new SQLQueryAdapter("SELECT proname, provolatile FROM pg_proc;"); + SQLancerResultSet rs = query.executeAndGet(globalState); + while (rs.next()) { + String functionName = rs.getString(1); + Character functionType = rs.getString(2).charAt(0); + globalState.addFunctionAndType(functionName, functionType); + } + } + + protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { + synchronized (CREATION_LOCK) { + boolean prevCreationFailed = false; // small optimization - wait only after failed requests + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + if (!prevCreationFailed) { + exceptionLessSleep(5000); + } + + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = YSQLTableGenerator.generate(tableName, generateOnlyKnown, + globalState); + globalState.executeStatement(createTable); + prevCreationFailed = false; + } catch (IgnoreMeException e) { + prevCreationFailed = true; + } + } + } + } + + private void exceptionLessSleep(long timeout) { + try { + Thread.sleep(timeout); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + + protected void prepareTables(YSQLGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + YSQLProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 15000;\n")); + } + + private String getCreateDatabaseCommand(YSQLGlobalState state) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE DATABASE ").append(databaseName).append(" "); + if (Randomly.getBoolean() && state.getDbmsSpecificOptions().testCollations) { + sb.append("WITH "); + if (Randomly.getBoolean()) { + sb.append("ENCODING '"); + sb.append(Randomly.fromOptions("utf8")); + sb.append("' "); + } + // disabled due to https://github.com/yugabyte/yugabyte-db/issues/11357 + // if (Randomly.getBoolean()) { + // sb.append("COLOCATED = true "); + // } + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } + } + sb.append(" TEMPLATE template0"); + + } + return sb.toString(); + } + + public enum Action implements AbstractAction { + ANALYZE(YSQLAnalyzeGenerator::create), // + ALTER_TABLE(g -> YSQLAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g)), // + COMMIT(g -> { + SQLQueryAdapter query; + if (Randomly.getBoolean()) { + query = new SQLQueryAdapter("COMMIT", true); + } else if (Randomly.getBoolean()) { + query = YSQLTransactionGenerator.executeBegin(); + } else { + query = new SQLQueryAdapter("ROLLBACK", true); + } + return query; + }), // + DELETE(YSQLDeleteGenerator::create), // + DISCARD(YSQLDiscardGenerator::create), // + DROP_INDEX(YSQLDropIndexGenerator::create), // + CREATE_INDEX(YSQLIndexGenerator::generate), // + INSERT(YSQLInsertGenerator::insert), // + UPDATE(YSQLUpdateGenerator::create), // + TRUNCATE(YSQLTruncateGenerator::create), // + TABLEGROUP(YSQLTableGroupGenerator::create), // + VACUUM(YSQLVacuumGenerator::create), // + SET(YSQLSetGenerator::create), // TODO insert yugabyte sets + SET_CONSTRAINTS((g) -> { + String sb = "SET CONSTRAINTS ALL " + Randomly.fromOptions("DEFERRED", "IMMEDIATE"); + return new SQLQueryAdapter(sb); + }), // + RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // + COMMENT_ON(YSQLCommentGenerator::generate), // + RESET((g) -> new SQLQueryAdapter("RESET ALL") /* + * https://www.postgres.org/docs/devel/sql-reset.html TODO: also + * configuration parameter + */), // + NOTIFY(YSQLNotifyGenerator::createNotify), // + LISTEN((g) -> YSQLNotifyGenerator.createListen()), // + UNLISTEN((g) -> YSQLNotifyGenerator.createUnlisten()), // + CREATE_SEQUENCE(YSQLSequenceGenerator::createSequence), // + CREATE_VIEW(YSQLViewGenerator::create); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(YSQLGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLSchema.java b/src/sqlancer/yugabyte/ysql/YSQLSchema.java new file mode 100644 index 000000000..6aba5ac7d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLSchema.java @@ -0,0 +1,329 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.postgresql.util.PSQLException; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; + +public class YSQLSchema extends AbstractSchema { + + private final String databaseName; + + public YSQLSchema(List databaseTables, String databaseName) { + super(databaseTables); + this.databaseName = databaseName; + } + + public static YSQLDataType getColumnType(String typeString) { + switch (typeString) { + case "smallint": + case "integer": + case "bigint": + return YSQLDataType.INT; + case "boolean": + return YSQLDataType.BOOLEAN; + case "text": + case "character": + case "character varying": + case "name": + return YSQLDataType.TEXT; + case "numeric": + return YSQLDataType.DECIMAL; + case "double precision": + return YSQLDataType.FLOAT; + case "real": + return YSQLDataType.REAL; + case "int4range": + return YSQLDataType.RANGE; + case "money": + return YSQLDataType.MONEY; + case "bytea": + return YSQLDataType.BYTEA; + case "bit": + case "bit varying": + return YSQLDataType.BIT; + case "inet": + return YSQLDataType.INET; + default: + throw new AssertionError(typeString); + } + } + + public static YSQLSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + try { + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + "SELECT table_name, table_schema, table_type, is_insertable_into FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + String tableTypeSchema = rs.getString("table_schema"); + boolean isInsertable = rs.getBoolean("is_insertable_into"); + // TODO: also check insertable + // TODO: insert into view? + boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || + // tableTypeStr.contains("LOCAL TEMPORARY") && + // !isInsertable; + YSQLTable.TableType tableType = getTableType(tableTypeSchema); + List databaseColumns = getTableColumns(con, tableName); + List indexes = getIndexes(con, tableName); + List statistics = getStatistics(con); + YSQLTable t = new YSQLTable(tableName, databaseColumns, indexes, tableType, statistics, isView, + isInsertable); + for (YSQLColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + return new YSQLSchema(databaseTables, databaseName); + } catch (SQLIntegrityConstraintViolationException e) { + throw new AssertionError(e); + } + } + + protected static List getStatistics(SQLConnection con) throws SQLException { + List statistics = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT stxname FROM pg_statistic_ext ORDER BY stxname;")) { + while (rs.next()) { + statistics.add(new YSQLStatisticsObject(rs.getString("stxname"))); + } + } + } + return statistics; + } + + protected static YSQLTable.TableType getTableType(String tableTypeStr) throws AssertionError { + YSQLTable.TableType tableType; + if (tableTypeStr.contentEquals("public")) { + tableType = YSQLTable.TableType.STANDARD; + } else if (tableTypeStr.startsWith("pg_temp")) { + tableType = YSQLTable.TableType.TEMPORARY; + } else { + throw new AssertionError(tableTypeStr); + } + return tableType; + } + + protected static List getIndexes(SQLConnection con, String tableName) throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String + .format("SELECT indexname FROM pg_indexes WHERE tablename='%s' ORDER BY indexname;", tableName))) { + while (rs.next()) { + String indexName = rs.getString("indexname"); + if (DBMSCommon.matchesIndexName(indexName)) { + indexes.add(YSQLIndex.create(indexName)); + } + } + } + } + return indexes; + } + + protected static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("select column_name, data_type from INFORMATION_SCHEMA.COLUMNS where table_name = '" + + tableName + "' ORDER BY column_name")) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + YSQLColumn c = new YSQLColumn(columnName, getColumnType(dataType)); + columns.add(c); + } + } + } + return columns; + } + + public YSQLTables getRandomTableNonEmptyTables() { + return new YSQLTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public String getDatabaseName() { + return databaseName; + } + + public enum YSQLDataType { + // TODO: 23.02.2022 Planned types + // SMALLINT, INT, BIGINT, NUMERIC, DECIMAL, REAL, DOUBLE_PRECISION, VARCHAR, CHAR, TEXT, DATE, TIME, + // TIMESTAMP, TIMESTAMPZ, INTERVAL, INTEGER_ARR + INT, BOOLEAN, BYTEA, TEXT, DECIMAL, FLOAT, REAL, RANGE, MONEY, BIT, INET; + + public static YSQLDataType getRandomType() { + List dataTypes = new ArrayList<>(Arrays.asList(values())); + if (YSQLProvider.generateOnlyKnown) { + dataTypes.remove(YSQLDataType.DECIMAL); + dataTypes.remove(YSQLDataType.FLOAT); + dataTypes.remove(YSQLDataType.REAL); + dataTypes.remove(YSQLDataType.INET); + dataTypes.remove(YSQLDataType.RANGE); + dataTypes.remove(YSQLDataType.MONEY); + dataTypes.remove(YSQLDataType.BIT); + } + return Randomly.fromList(dataTypes); + } + } + + public static class YSQLColumn extends AbstractTableColumn { + + public YSQLColumn(String name, YSQLDataType columnType) { + super(name, null, columnType); + } + + public static YSQLColumn createDummy(String name) { + return new YSQLColumn(name, YSQLDataType.INT); + } + + } + + public static class YSQLTables extends AbstractTables { + + public YSQLTables(List tables) { + super(tables); + } + + public YSQLRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String randomRow = String.format("SELECT %s FROM %s ORDER BY RANDOM() LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + // columnNamesAsString(c -> "typeof(" + c.getTable().getName() + "." + + // c.getName() + ")") + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet randomRowValues = s.executeQuery(randomRow); + if (!randomRowValues.next()) { + throw new AssertionError("could not find random row! " + randomRow + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + YSQLColumn column = getColumns().get(i); + int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + YSQLConstant constant; + if (randomRowValues.getString(columnIndex) == null) { + constant = YSQLConstant.createNullConstant(); + } else { + switch (column.getType()) { + case INT: + constant = YSQLConstant.createIntConstant(randomRowValues.getLong(columnIndex)); + break; + case BOOLEAN: + constant = YSQLConstant.createBooleanConstant(randomRowValues.getBoolean(columnIndex)); + break; + case TEXT: + constant = YSQLConstant.createTextConstant(randomRowValues.getString(columnIndex)); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !randomRowValues.next(); + return new YSQLRowValue(this, values); + } catch (PSQLException e) { + throw new IgnoreMeException(); + } + + } + + } + + public static class YSQLRowValue extends AbstractRowValue { + + protected YSQLRowValue(YSQLTables tables, Map values) { + super(tables, values); + } + + } + + public static class YSQLTable extends AbstractRelationalTable { + + private final TableType tableType; + private final List statistics; + private final boolean isInsertable; + + public YSQLTable(String tableName, List columns, List indexes, TableType tableType, + List statistics, boolean isView, boolean isInsertable) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + } + + public List getStatistics() { + return statistics; + } + + public TableType getTableType() { + return tableType; + } + + public boolean isInsertable() { + return isInsertable; + } + + public enum TableType { + STANDARD, TEMPORARY + } + + } + + public static final class YSQLStatisticsObject { + private final String name; + + public YSQLStatisticsObject(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + public static final class YSQLIndex extends TableIndex { + + private YSQLIndex(String indexName) { + super(indexName); + } + + public static YSQLIndex create(String indexName) { + return new YSQLIndex(indexName); + } + + @Override + public String getIndexName() { + if (super.getIndexName().contentEquals("PRIMARY")) { + return "`PRIMARY`"; + } else { + return super.getIndexName(); + } + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java new file mode 100644 index 000000000..39732b728 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java @@ -0,0 +1,329 @@ +package sqlancer.yugabyte.ysql; + +import java.util.Optional; + +import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLJoin.YSQLJoinType; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLSubquery; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; + +public final class YSQLToStringVisitor extends ToStringVisitor implements YSQLVisitor { + + @Override + public void visitSpecific(YSQLExpression expr) { + YSQLVisitor.super.visit(expr); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(YSQLConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public void visit(YSQLPostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" "); + sb.append(op.getOperatorTextRepresentation()); + } + + @Override + public void visit(YSQLColumnValue c) { + sb.append(c.getColumn().getFullQualifiedName()); + } + + @Override + public void visit(YSQLPrefixOperation op) { + sb.append(op.getTextRepresentation()); + sb.append(" ("); + visit(op.getExpression()); + sb.append(")"); + } + + @Override + public void visit(YSQLSelect s) { + sb.append("SELECT "); + switch (s.getSelectOption()) { + case DISTINCT: + sb.append("DISTINCT "); + if (s.getDistinctOnClause() != null) { + sb.append("ON ("); + visit(s.getDistinctOnClause()); + sb.append(") "); + } + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + if (s.getFetchColumns() == null) { + sb.append("*"); + } else { + visit(s.getFetchColumns()); + } + sb.append(" FROM "); + visit(s.getFromList()); + + for (YSQLJoin j : s.getJoinClauses()) { + sb.append(" "); + switch (j.getType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } + sb.append("JOIN"); + break; + case LEFT: + sb.append("LEFT OUTER JOIN"); + break; + case RIGHT: + sb.append("RIGHT OUTER JOIN"); + break; + case FULL: + sb.append("FULL OUTER JOIN"); + break; + case CROSS: + sb.append("CROSS JOIN"); + break; + default: + throw new AssertionError(j.getType()); + } + sb.append(" "); + visit(j.getTableReference()); + if (j.getType() != YSQLJoinType.CROSS) { + sb.append(" ON "); + visit(j.getOnClause()); + } + } + + if (s.getWhereClause() != null) { + sb.append(" WHERE "); + visit(s.getWhereClause()); + } + if (s.getGroupByExpressions().size() > 0) { + sb.append(" GROUP BY "); + visit(s.getGroupByExpressions()); + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + + } + if (!s.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(s.getOrderByExpressions()); + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(YSQLOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder()); + } + + @Override + public void visit(YSQLFunction f) { + sb.append(f.getFunctionName()); + sb.append("("); + int i = 0; + for (YSQLExpression arg : f.getArguments()) { + if (i++ != 0) { + sb.append(", "); + } + visit(arg); + } + sb.append(")"); + } + + @Override + public void visit(YSQLCastOperation cast) { + if (Randomly.getBoolean()) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + appendType(cast); + sb.append(")"); + } else { + sb.append("("); + visit(cast.getExpression()); + sb.append(")::"); + appendType(cast); + } + } + + @Override + public void visit(YSQLBetweenOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(") BETWEEN "); + if (op.isSymmetric()) { + sb.append("SYMMETRIC "); + } + sb.append("("); + visit(op.getLeft()); + sb.append(") AND ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(YSQLInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN ("); + visit(op.getListElements()); + sb.append(")"); + } + + @Override + public void visit(YSQLPostfixText op) { + visit(op.getExpr()); + sb.append(op.getText()); + } + + @Override + public void visit(YSQLAggregate op) { + sb.append(op.getFunction()); + sb.append("("); + visit(op.getArgs()); + sb.append(")"); + } + + @Override + public void visit(YSQLSimilarTo op) { + sb.append("("); + visit(op.getString()); + sb.append(" SIMILAR TO "); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + sb.append(")"); + } + + @Override + public void visit(YSQLPOSIXRegularExpression op) { + visit(op.getString()); + sb.append(op.getOp().getStringRepresentation()); + visit(op.getRegex()); + } + + @Override + public void visit(YSQLFromTable from) { + if (from.isOnly()) { + sb.append("ONLY "); + } + sb.append(from.getTable().getName()); + if (!from.isOnly() && Randomly.getBoolean()) { + sb.append("*"); + } + } + + @Override + public void visit(YSQLSubquery subquery) { + sb.append("("); + visit(subquery.getSelect()); + sb.append(") AS "); + sb.append(subquery.getName()); + } + + @Override + public void visit(YSQLBinaryLogicalOperation op) { + super.visit((BinaryOperation) op); + } + + private void appendType(YSQLCastOperation cast) { + YSQLCompoundDataType compoundType = cast.getCompoundType(); + switch (compoundType.getDataType()) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: // TODO support also other int types + sb.append("INT"); + break; + case TEXT: + // TODO: append TEXT, CHAR + sb.append(Randomly.fromOptions("VARCHAR")); + break; + case REAL: + sb.append("REAL"); + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + sb.append("FLOAT"); + break; + case RANGE: + sb.append("int4range"); + break; + case MONEY: + sb.append("MONEY"); + break; + case INET: + sb.append("INET"); + break; + case BIT: + sb.append("BIT"); + break; + case BYTEA: + sb.append("BYTEA"); + break; + // if (Randomly.getBoolean()) { + // sb.append("("); + // sb.append(Randomly.getNotCachedInteger(1, 100)); + // sb.append(")"); + // } + default: + throw new AssertionError(cast.getType()); + } + Optional size = compoundType.getSize(); + if (size.isPresent()) { + sb.append("("); + sb.append(size.get()); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLVisitor.java new file mode 100644 index 000000000..a73aade4c --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLVisitor.java @@ -0,0 +1,120 @@ +package sqlancer.yugabyte.ysql; + +import java.util.List; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLSubquery; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public interface YSQLVisitor { + + static String asString(YSQLExpression expr) { + YSQLToStringVisitor visitor = new YSQLToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + static String asExpectedValues(YSQLExpression expr) { + YSQLExpectedValueVisitor v = new YSQLExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + + static String getExpressionAsString(YSQLGlobalState globalState, YSQLDataType type, List columns) { + YSQLExpression expression = YSQLExpressionGenerator.generateExpression(globalState, columns, type); + YSQLToStringVisitor visitor = new YSQLToStringVisitor(); + visitor.visit(expression); + return visitor.get(); + } + + void visit(YSQLConstant constant); + + void visit(YSQLPostfixOperation op); + + void visit(YSQLColumnValue c); + + void visit(YSQLPrefixOperation op); + + void visit(YSQLSelect op); + + void visit(YSQLOrderByTerm op); + + void visit(YSQLFunction f); + + void visit(YSQLCastOperation cast); + + void visit(YSQLBetweenOperation op); + + void visit(YSQLInOperation op); + + void visit(YSQLPostfixText op); + + void visit(YSQLAggregate op); + + void visit(YSQLSimilarTo op); + + void visit(YSQLPOSIXRegularExpression op); + + void visit(YSQLFromTable from); + + void visit(YSQLSubquery subquery); + + void visit(YSQLBinaryLogicalOperation op); + + default void visit(YSQLExpression expression) { + if (expression instanceof YSQLConstant) { + visit((YSQLConstant) expression); + } else if (expression instanceof YSQLPostfixOperation) { + visit((YSQLPostfixOperation) expression); + } else if (expression instanceof YSQLColumnValue) { + visit((YSQLColumnValue) expression); + } else if (expression instanceof YSQLPrefixOperation) { + visit((YSQLPrefixOperation) expression); + } else if (expression instanceof YSQLSelect) { + visit((YSQLSelect) expression); + } else if (expression instanceof YSQLOrderByTerm) { + visit((YSQLOrderByTerm) expression); + } else if (expression instanceof YSQLFunction) { + visit((YSQLFunction) expression); + } else if (expression instanceof YSQLCastOperation) { + visit((YSQLCastOperation) expression); + } else if (expression instanceof YSQLBetweenOperation) { + visit((YSQLBetweenOperation) expression); + } else if (expression instanceof YSQLInOperation) { + visit((YSQLInOperation) expression); + } else if (expression instanceof YSQLAggregate) { + visit((YSQLAggregate) expression); + } else if (expression instanceof YSQLPostfixText) { + visit((YSQLPostfixText) expression); + } else if (expression instanceof YSQLSimilarTo) { + visit((YSQLSimilarTo) expression); + } else if (expression instanceof YSQLPOSIXRegularExpression) { + visit((YSQLPOSIXRegularExpression) expression); + } else if (expression instanceof YSQLFromTable) { + visit((YSQLFromTable) expression); + } else if (expression instanceof YSQLSubquery) { + visit((YSQLSubquery) expression); + } else { + throw new AssertionError(expression); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java b/src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java new file mode 100644 index 000000000..27daefcf9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLAggregate.java @@ -0,0 +1,58 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate.YSQLAggregateFunction; + +/** + * @see Built-in Aggregate Functions + */ +public class YSQLAggregate extends FunctionNode implements YSQLExpression { + + public YSQLAggregate(List args, YSQLAggregateFunction func) { + super(func, args); + } + + public enum YSQLAggregateFunction { + AVG(YSQLDataType.INT, YSQLDataType.FLOAT, YSQLDataType.REAL, YSQLDataType.DECIMAL), BIT_AND(YSQLDataType.INT), + BIT_OR(YSQLDataType.INT), BOOL_AND(YSQLDataType.BOOLEAN), BOOL_OR(YSQLDataType.BOOLEAN), + COUNT(YSQLDataType.INT), EVERY(YSQLDataType.BOOLEAN), MAX, MIN, + // STRING_AGG + SUM(YSQLDataType.INT, YSQLDataType.FLOAT, YSQLDataType.REAL, YSQLDataType.DECIMAL); + + private final YSQLDataType[] supportedReturnTypes; + + YSQLAggregateFunction(YSQLDataType... supportedReturnTypes) { + this.supportedReturnTypes = supportedReturnTypes.clone(); + } + + public static List getAggregates(YSQLDataType type) { + return Arrays.stream(values()).filter(p -> p.supportsReturnType(type)).collect(Collectors.toList()); + } + + public List getTypes(YSQLDataType returnType) { + return Collections.singletonList(returnType); + } + + public boolean supportsReturnType(YSQLDataType returnType) { + return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType) + || supportedReturnTypes.length == 0; + } + + public YSQLDataType getRandomReturnType() { + if (supportedReturnTypes.length == 0) { + return Randomly.fromOptions(YSQLDataType.getRandomType()); + } else { + return Randomly.fromOptions(supportedReturnTypes); + } + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java b/src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java new file mode 100644 index 000000000..96e432370 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLAlias.java @@ -0,0 +1,35 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.common.visitor.UnaryOperation; + +public class YSQLAlias implements UnaryOperation, YSQLExpression { + + private final YSQLExpression expr; + private final String alias; + + public YSQLAlias(YSQLExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + @Override + public YSQLExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return " as " + alias; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java new file mode 100644 index 000000000..4a6c123d0 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBetweenOperation.java @@ -0,0 +1,63 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public final class YSQLBetweenOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final YSQLExpression left; + private final YSQLExpression right; + private final boolean isSymmetric; + + public YSQLBetweenOperation(YSQLExpression expr, YSQLExpression left, YSQLExpression right, boolean symmetric) { + this.expr = expr; + this.left = left; + this.right = right; + isSymmetric = symmetric; + } + + public YSQLExpression getExpr() { + return expr; + } + + public YSQLExpression getLeft() { + return left; + } + + public YSQLExpression getRight() { + return right; + } + + public boolean isSymmetric() { + return isSymmetric; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLBinaryComparisonOperation leftComparison = new YSQLBinaryComparisonOperation(left, expr, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryComparisonOperation rightComparison = new YSQLBinaryComparisonOperation(expr, right, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryLogicalOperation andOperation = new YSQLBinaryLogicalOperation(leftComparison, rightComparison, + YSQLBinaryLogicalOperation.BinaryLogicalOperator.AND); + if (isSymmetric) { + YSQLBinaryComparisonOperation leftComparison2 = new YSQLBinaryComparisonOperation(right, expr, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryComparisonOperation rightComparison2 = new YSQLBinaryComparisonOperation(expr, left, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.LESS_EQUALS); + YSQLBinaryLogicalOperation andOperation2 = new YSQLBinaryLogicalOperation(leftComparison2, rightComparison2, + YSQLBinaryLogicalOperation.BinaryLogicalOperator.AND); + YSQLBinaryLogicalOperation orOp = new YSQLBinaryLogicalOperation(andOperation, andOperation2, + YSQLBinaryLogicalOperation.BinaryLogicalOperator.OR); + return orOp.getExpectedValue(); + } else { + return andOperation.getExpectedValue(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java new file mode 100644 index 000000000..a4385c86c --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryArithmeticOperation.java @@ -0,0 +1,106 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryArithmeticOperation.YSQLBinaryOperator; + +public class YSQLBinaryArithmeticOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryArithmeticOperation(YSQLExpression left, YSQLExpression right, YSQLBinaryOperator op) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpected = getLeft().getExpectedValue(); + YSQLConstant rightExpected = getRight().getExpectedValue(); + if (leftExpected == null || rightExpected == null) { + return null; + } + return getOp().apply(leftExpected, rightExpected); + } + + public enum YSQLBinaryOperator implements Operator { + + ADDITION("+") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, Long::sum); + } + + }, + SUBTRACTION("-") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + + } + + }, + MODULO("%") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + + } + }, + EXPONENTIATION("^") { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + return null; + } + }; + + private final String textRepresentation; + + YSQLBinaryOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + private static YSQLConstant applyBitOperation(YSQLConstant left, YSQLConstant right, BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + long leftVal = left.cast(YSQLDataType.INT).asInt(); + long rightVal = right.cast(YSQLDataType.INT).asInt(); + long value = op.apply(leftVal, rightVal); + return YSQLConstant.createIntConstant(value); + } + } + + public static YSQLBinaryOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public abstract YSQLConstant apply(YSQLConstant left, YSQLConstant right); + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java new file mode 100644 index 000000000..0d1cb8f2e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryBitOperation.java @@ -0,0 +1,46 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryBitOperation.YSQLBinaryBitOperator; + +public class YSQLBinaryBitOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryBitOperation(YSQLBinaryBitOperator op, YSQLExpression left, YSQLExpression right) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BIT; + } + + public enum YSQLBinaryBitOperator implements Operator { + CONCATENATION("||"), // + BITWISE_AND("&"), // + BITWISE_OR("|"), // + BITWISE_XOR("#"), // + BITWISE_SHIFT_LEFT("<<"), // + BITWISE_SHIFT_RIGHT(">>"); + + private final String text; + + YSQLBinaryBitOperator(String text) { + this.text = text; + } + + public static YSQLBinaryBitOperator getRandom() { + return Randomly.fromOptions(YSQLBinaryBitOperator.values()); + } + + @Override + public String getTextRepresentation() { + return text; + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java new file mode 100644 index 000000000..8ced603ac --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryComparisonOperation.java @@ -0,0 +1,135 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator; + +public class YSQLBinaryComparisonOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryComparisonOperation(YSQLExpression left, YSQLExpression right, YSQLBinaryComparisonOperator op) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpectedValue = getLeft().getExpectedValue(); + YSQLConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); + } + + public enum YSQLBinaryComparisonOperator implements Operator { + EQUALS("=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + return leftVal.isEquals(rightVal); + } + }, + IS_DISTINCT("IS DISTINCT FROM") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + return YSQLConstant + .createBooleanConstant(!IS_NOT_DISTINCT.getExpectedValue(leftVal, rightVal).asBoolean()); + } + }, + IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + if (leftVal.isNull()) { + return YSQLConstant.createBooleanConstant(rightVal.isNull()); + } else if (rightVal.isNull()) { + return YSQLConstant.createFalse(); + } else { + return leftVal.isEquals(rightVal); + } + } + }, + NOT_EQUALS("!=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant isEquals = leftVal.isEquals(rightVal); + if (isEquals.isBoolean()) { + return YSQLConstant.createBooleanConstant(!isEquals.asBoolean()); + } + return isEquals; + } + }, + LESS("<") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + return leftVal.isLessThan(rightVal); + } + }, + LESS_EQUALS("<=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant lessThan = leftVal.isLessThan(rightVal); + if (lessThan.isBoolean() && !lessThan.asBoolean()) { + return leftVal.isEquals(rightVal); + } else { + return lessThan; + } + } + }, + GREATER(">") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return YSQLConstant.createFalse(); + } else { + YSQLConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return YSQLConstant.createNullConstant(); + } + return YSQLPrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + }, + GREATER_EQUALS(">=") { + @Override + public YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal) { + YSQLConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return YSQLConstant.createTrue(); + } else { + YSQLConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return YSQLConstant.createNullConstant(); + } + return YSQLPrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + + }; + + private final String textRepresentation; + + YSQLBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static YSQLBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(YSQLBinaryComparisonOperator.values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public abstract YSQLConstant getExpectedValue(YSQLConstant leftVal, YSQLConstant rightVal); + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java new file mode 100644 index 000000000..89cce762f --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryLogicalOperation.java @@ -0,0 +1,88 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation.BinaryLogicalOperator; + +public class YSQLBinaryLogicalOperation extends BinaryOperatorNode + implements YSQLExpression { + + public YSQLBinaryLogicalOperation(YSQLExpression left, YSQLExpression right, BinaryLogicalOperator op) { + super(left, right, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpectedValue = getLeft().getExpectedValue(); + YSQLConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + + public enum BinaryLogicalOperator implements Operator { + AND { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + YSQLConstant leftBool = left.cast(YSQLDataType.BOOLEAN); + YSQLConstant rightBool = right.cast(YSQLDataType.BOOLEAN); + if (leftBool.isNull()) { + if (rightBool.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + if (rightBool.asBoolean()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createFalse(); + } + } + } else if (!leftBool.asBoolean()) { + return YSQLConstant.createFalse(); + } + assert leftBool.asBoolean(); + if (rightBool.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createBooleanConstant(rightBool.isBoolean() && rightBool.asBoolean()); + } + } + }, + OR { + @Override + public YSQLConstant apply(YSQLConstant left, YSQLConstant right) { + YSQLConstant leftBool = left.cast(YSQLDataType.BOOLEAN); + YSQLConstant rightBool = right.cast(YSQLDataType.BOOLEAN); + if (leftBool.isBoolean() && leftBool.asBoolean()) { + return YSQLConstant.createTrue(); + } + if (rightBool.isBoolean() && rightBool.asBoolean()) { + return YSQLConstant.createTrue(); + } + if (leftBool.isNull() || rightBool.isNull()) { + return YSQLConstant.createNullConstant(); + } + return YSQLConstant.createFalse(); + } + }; + + public static BinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract YSQLConstant apply(YSQLConstant left, YSQLConstant right); + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java new file mode 100644 index 000000000..4bf4a8ea6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLBinaryRangeOperation.java @@ -0,0 +1,71 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLBinaryRangeOperation extends BinaryNode implements YSQLExpression { + + private final String op; + + public YSQLBinaryRangeOperation(YSQLBinaryRangeComparisonOperator op, YSQLExpression left, YSQLExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + public YSQLBinaryRangeOperation(YSQLBinaryRangeOperator op, YSQLExpression left, YSQLExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public String getOperatorRepresentation() { + return op; + } + + public enum YSQLBinaryRangeOperator implements Operator { + UNION("+"), INTERSECTION("*"), DIFFERENCE("-"); + + private final String textRepresentation; + + YSQLBinaryRangeOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static YSQLBinaryRangeOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum YSQLBinaryRangeComparisonOperator { + CONTAINS_RANGE_OR_ELEMENT("@>"), RANGE_OR_ELEMENT_IS_CONTAINED("<@"), OVERLAP("&&"), STRICT_LEFT_OF("<<"), + STRICT_RIGHT_OF(">>"), NOT_RIGHT_OF("&<"), NOT_LEFT_OF(">&"), ADJACENT("-|-"); + + private final String textRepresentation; + + YSQLBinaryRangeComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static YSQLBinaryRangeComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java new file mode 100644 index 000000000..cacd8ad76 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLCastOperation.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLCastOperation implements YSQLExpression { + + private final YSQLExpression expression; + private final YSQLCompoundDataType type; + + public YSQLCastOperation(YSQLExpression expression, YSQLCompoundDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + @Override + public YSQLDataType getExpressionType() { + return type.getDataType(); + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant expectedValue = expression.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type.getDataType()); + } + + public YSQLExpression getExpression() { + return expression; + } + + public YSQLDataType getType() { + return type.getDataType(); + } + + public YSQLCompoundDataType getCompoundType() { + return type; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java b/src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java new file mode 100644 index 000000000..82b164c54 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLCollate.java @@ -0,0 +1,33 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLCollate implements YSQLExpression { + + private final YSQLExpression expr; + private final String collate; + + public YSQLCollate(YSQLExpression expr, String collate) { + this.expr = expr; + this.collate = collate; + } + + public String getCollate() { + return collate; + } + + public YSQLExpression getExpr() { + return expr; + } + + @Override + public YSQLDataType getExpressionType() { + return expr.getExpressionType(); + } + + @Override + public YSQLConstant getExpectedValue() { + return null; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java b/src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java new file mode 100644 index 000000000..243bdaf57 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLColumnValue.java @@ -0,0 +1,34 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLColumnValue implements YSQLExpression { + + private final YSQLColumn c; + private final YSQLConstant expectedValue; + + public YSQLColumnValue(YSQLColumn c, YSQLConstant expectedValue) { + this.c = c; + this.expectedValue = expectedValue; + } + + public static YSQLColumnValue create(YSQLColumn c, YSQLConstant expected) { + return new YSQLColumnValue(c, expected); + } + + @Override + public YSQLDataType getExpressionType() { + return c.getType(); + } + + @Override + public YSQLConstant getExpectedValue() { + return expectedValue; + } + + public YSQLColumn getColumn() { + return c; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java new file mode 100644 index 000000000..a74263b22 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLConcatOperation.java @@ -0,0 +1,37 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.common.ast.BinaryNode; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLConcatOperation extends BinaryNode implements YSQLExpression { + + public YSQLConcatOperation(YSQLExpression left, YSQLExpression right) { + super(left, right); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.TEXT; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftExpectedValue = getLeft().getExpectedValue(); + YSQLConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + if (leftExpectedValue.isNull() || rightExpectedValue.isNull()) { + return YSQLConstant.createNullConstant(); + } + String leftStr = leftExpectedValue.cast(YSQLDataType.TEXT).getUnquotedTextRepresentation(); + String rightStr = rightExpectedValue.cast(YSQLDataType.TEXT).getUnquotedTextRepresentation(); + return YSQLConstant.createTextConstant(leftStr + rightStr); + } + + @Override + public String getOperatorRepresentation() { + return "||"; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java b/src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java new file mode 100644 index 000000000..9fd2eaad6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLConstant.java @@ -0,0 +1,611 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.math.BigDecimal; + +import sqlancer.IgnoreMeException; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public abstract class YSQLConstant implements YSQLExpression { + + public static YSQLConstant createNullConstant() { + return new YSQLNullConstant(); + } + + public static YSQLConstant createIntConstant(long val) { + return new IntConstant(val); + } + + public static YSQLConstant createBooleanConstant(boolean val) { + return new BooleanConstant(val); + } + + public static YSQLConstant createFalse() { + return createBooleanConstant(false); + } + + public static YSQLConstant createTrue() { + return createBooleanConstant(true); + } + + public static YSQLConstant createTextConstant(String string) { + return new StringConstant(string); + } + + public static YSQLConstant createByteConstant(String string) { + return new ByteConstant(string); + } + + public static YSQLConstant createDecimalConstant(BigDecimal bigDecimal) { + return new DecimalConstant(bigDecimal); + } + + public static YSQLConstant createFloatConstant(float val) { + return new FloatConstant(val); + } + + public static YSQLConstant createDoubleConstant(double val) { + return new DoubleConstant(val); + } + + public static YSQLConstant createRange(long left, boolean leftIsInclusive, long right, boolean rightIsInclusive) { + long realLeft; + long realRight; + if (left > right) { + realRight = left; + realLeft = right; + } else { + realLeft = left; + realRight = right; + } + return new RangeConstant(realLeft, leftIsInclusive, realRight, rightIsInclusive); + } + + public static YSQLExpression createBitConstant(long integer) { + return new BitConstant(integer); + } + + public static YSQLExpression createInetConstant(String val) { + return new InetConstant(val); + } + + public abstract String getTextRepresentation(); + + public abstract String getUnquotedTextRepresentation(); + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isString() { + return false; + } + + @Override + public YSQLConstant getExpectedValue() { + return this; + } + + public boolean isNull() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isBoolean() { + return false; + } + + public abstract YSQLConstant isEquals(YSQLConstant rightVal); + + public boolean isInt() { + return false; + } + + protected abstract YSQLConstant isLessThan(YSQLConstant rightVal); + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract YSQLConstant cast(YSQLDataType type); + + public static class BooleanConstant extends YSQLConstant { + + private final boolean value; + + public BooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return value ? "TRUE" : "FALSE"; + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return YSQLConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(value == rightVal.cast(YSQLDataType.BOOLEAN).asBoolean()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isString()) { + return isLessThan(rightVal.cast(YSQLDataType.BOOLEAN)); + } else { + assert rightVal.isBoolean(); + return YSQLConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + switch (type) { + case BOOLEAN: + return this; + case INT: + return YSQLConstant.createIntConstant(value ? 1 : 0); + case TEXT: + return YSQLConstant.createTextConstant(value ? "true" : "false"); + default: + return null; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + } + + public static class YSQLNullConstant extends YSQLConstant { + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + return YSQLConstant.createNullConstant(); + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + return YSQLConstant.createNullConstant(); + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + return YSQLConstant.createNullConstant(); + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + + } + + public static class StringConstant extends YSQLConstant { + + protected final String value; + + public StringConstant(String value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'", value.replace("'", "''")); + } + + @Override + public String getUnquotedTextRepresentation() { + return value; + } + + @Override + public String asString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(YSQLDataType.INT).isEquals(rightVal.cast(YSQLDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(YSQLDataType.BOOLEAN).isEquals(rightVal.cast(YSQLDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(YSQLDataType.INT).isLessThan(rightVal.cast(YSQLDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(YSQLDataType.BOOLEAN).isLessThan(rightVal.cast(YSQLDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + if (type == YSQLDataType.TEXT) { + return this; + } + String s = value.trim(); + switch (type) { + case BOOLEAN: + try { + return YSQLConstant.createBooleanConstant(Long.parseLong(s) != 0); + } catch (NumberFormatException e) { + } + switch (s.toUpperCase()) { + case "T": + case "TR": + case "TRU": + case "TRUE": + case "1": + case "YES": + case "YE": + case "Y": + case "ON": + return YSQLConstant.createTrue(); + case "F": + case "FA": + case "FAL": + case "FALS": + case "FALSE": + case "N": + case "NO": + case "OF": + case "OFF": + default: + return YSQLConstant.createFalse(); + } + case INT: + try { + return YSQLConstant.createIntConstant(Long.parseLong(s)); + } catch (NumberFormatException e) { + return YSQLConstant.createIntConstant(-1); + } + case TEXT: + return this; + default: + return null; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.TEXT; + } + + } + + public static class IntConstant extends YSQLConstant { + + private final long val; + + public IntConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + @Override + public long asInt() { + return val; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(YSQLDataType.BOOLEAN).isEquals(rightVal); + } else if (rightVal.isInt()) { + return YSQLConstant.createBooleanConstant(val == rightVal.asInt()); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(val == rightVal.cast(YSQLDataType.INT).asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public boolean isInt() { + return true; + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + if (rightVal.isNull()) { + return YSQLConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return YSQLConstant.createBooleanConstant(val < rightVal.asInt()); + } else if (rightVal.isBoolean()) { + throw new AssertionError(rightVal); + } else if (rightVal.isString()) { + return YSQLConstant.createBooleanConstant(val < rightVal.cast(YSQLDataType.INT).asInt()); + } else { + throw new IgnoreMeException(); + } + + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + switch (type) { + case BOOLEAN: + return YSQLConstant.createBooleanConstant(val != 0); + case INT: + return this; + case TEXT: + return YSQLConstant.createTextConstant(String.valueOf(val)); + default: + return null; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + } + + public static class ByteConstant extends StringConstant { + + public ByteConstant(String value) { + super(value); + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'::bytea", value.replace("'", "''")); + } + } + + public abstract static class YSQLConstantBase extends YSQLConstant { + + @Override + public String getUnquotedTextRepresentation() { + return null; + } + + @Override + public YSQLConstant isEquals(YSQLConstant rightVal) { + return null; + } + + @Override + protected YSQLConstant isLessThan(YSQLConstant rightVal) { + return null; + } + + @Override + public YSQLConstant cast(YSQLDataType type) { + return null; + } + } + + public static class DecimalConstant extends YSQLConstantBase { + + private final BigDecimal val; + + public DecimalConstant(BigDecimal val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.DECIMAL; + } + + } + + public static class InetConstant extends YSQLConstantBase { + + private final String val; + + public InetConstant(String val) { + this.val = "'" + val + "'"; + } + + @Override + public String getTextRepresentation() { + return val; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INET; + } + + } + + public static class FloatConstant extends YSQLConstantBase { + + private final float val; + + public FloatConstant(float val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.FLOAT; + } + + } + + public static class DoubleConstant extends YSQLConstantBase { + + private final double val; + + public DoubleConstant(double val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.FLOAT; + } + + } + + public static class BitConstant extends YSQLConstantBase { + + private final long val; + + public BitConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.format("B'%s'", Long.toBinaryString(val)); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BIT; + } + + } + + public static class RangeConstant extends YSQLConstantBase { + + private final long left; + private final boolean leftIsInclusive; + private final long right; + private final boolean rightIsInclusive; + + public RangeConstant(long left, boolean leftIsInclusive, long right, boolean rightIsInclusive) { + this.left = left; + this.leftIsInclusive = leftIsInclusive; + this.right = right; + this.rightIsInclusive = rightIsInclusive; + } + + @Override + public String getTextRepresentation() { + StringBuilder sb = new StringBuilder(); + sb.append("'"); + if (leftIsInclusive) { + sb.append("["); + } else { + sb.append("("); + } + sb.append(left); + sb.append(","); + sb.append(right); + if (rightIsInclusive) { + sb.append("]"); + } else { + sb.append(")"); + } + sb.append("'"); + sb.append("::int4range"); + return sb.toString(); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.RANGE; + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java new file mode 100644 index 000000000..9544c8838 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java @@ -0,0 +1,14 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public interface YSQLExpression { + + default YSQLDataType getExpressionType() { + return null; + } + + default YSQLConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java b/src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java new file mode 100644 index 000000000..54349ebe9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLFunction.java @@ -0,0 +1,283 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLFunction implements YSQLExpression { + + private final String func; + private final YSQLExpression[] args; + private final YSQLDataType returnType; + private YSQLFunctionWithResult functionWithKnownResult; + + public YSQLFunction(YSQLFunctionWithResult func, YSQLDataType returnType, YSQLExpression... args) { + functionWithKnownResult = func; + this.func = func.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public YSQLFunction(YSQLFunctionWithUnknownResult f, YSQLDataType returnType, YSQLExpression... args) { + this.func = f.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public String getFunctionName() { + return func; + } + + public YSQLExpression[] getArguments() { + return args.clone(); + } + + @Override + public YSQLDataType getExpressionType() { + return returnType; + } + + @Override + public YSQLConstant getExpectedValue() { + if (functionWithKnownResult == null) { + return null; + } + YSQLConstant[] constants = new YSQLConstant[args.length]; + for (int i = 0; i < constants.length; i++) { + constants[i] = args[i].getExpectedValue(); + if (constants[i] == null) { + return null; + } + } + return functionWithKnownResult.apply(constants, args); + } + + public enum YSQLFunctionWithResult { + ABS(1, "abs") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createIntConstant(Math.abs(evaluatedArgs[0].cast(YSQLDataType.INT).asInt())); + } + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { returnType }; + } + + }, + LOWER(1, "lower") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return YSQLConstant.createTextConstant(text.toLowerCase()); + } + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.TEXT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { YSQLDataType.TEXT }; + } + + }, + LENGTH(1, "length") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } + String text = evaluatedArgs[0].asString(); + return YSQLConstant.createIntConstant(text.length()); + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { YSQLDataType.TEXT }; + } + }, + UPPER(1, "upper") { + @Override + public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args) { + if (evaluatedArgs[0].isNull()) { + return YSQLConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return YSQLConstant.createTextConstant(text.toUpperCase()); + } + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.TEXT; + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return new YSQLDataType[] { YSQLDataType.TEXT }; + } + + }, + // NULL_IF(2, "nullif") { + // + // @Override + // public YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression[] args) { + // YSQLConstant equals = evaluatedArgs[0].isEquals(evaluatedArgs[1]); + // if (equals.isBoolean() && equals.asBoolean()) { + // return YSQLConstant.createNullConstant(); + // } else { + // // TODO: SELECT (nullif('1', FALSE)); yields '1', but should yield TRUE + // return evaluatedArgs[0]; + // } + // } + // + // @Override + // public boolean supportsReturnType(YSQLDataType type) { + // return true; + // } + // + // @Override + // public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + // return getType(nrArguments, returnType); + // } + // + // @Override + // public boolean checkArguments(YSQLExpression[] constants) { + // for (YSQLExpression e : constants) { + // if (!(e instanceof YSQLNullConstant)) { + // return true; + // } + // } + // return false; + // } + // + // }, + NUM_NONNULLS(1, "num_nonnulls") { + @Override + public YSQLConstant apply(YSQLConstant[] args, YSQLExpression... origArgs) { + int nr = 0; + for (YSQLConstant c : args) { + if (!c.isNull()) { + nr++; + } + } + return YSQLConstant.createIntConstant(nr); + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }, + NUM_NULLS(1, "num_nulls") { + @Override + public YSQLConstant apply(YSQLConstant[] args, YSQLExpression... origArgs) { + int nr = 0; + for (YSQLConstant c : args) { + if (c.isNull()) { + nr++; + } + } + return YSQLConstant.createIntConstant(nr); + } + + @Override + public YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(YSQLDataType type) { + return type == YSQLDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }; + + final int nrArgs; + private final String functionName; + private final boolean variadic; + + YSQLFunctionWithResult(int nrArgs, String functionName) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = false; + } + + public YSQLDataType[] getRandomTypes(int nr) { + YSQLDataType[] types = new YSQLDataType[nr]; + for (int i = 0; i < types.length; i++) { + types[i] = YSQLDataType.getRandomType(); + } + return types; + } + + /** + * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number + * of arguments is returned. + * + * @return the number of arguments + */ + public int getNrArgs() { + return nrArgs; + } + + public abstract YSQLConstant apply(YSQLConstant[] evaluatedArgs, YSQLExpression... args); + + @Override + public String toString() { + return functionName; + } + + public boolean isVariadic() { + return variadic; + } + + public String getName() { + return functionName; + } + + public abstract boolean supportsReturnType(YSQLDataType type); + + public abstract YSQLDataType[] getInputTypesForReturnType(YSQLDataType returnType, int nrArguments); + + public boolean checkArguments(YSQLExpression... constants) { + return true; + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java b/src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java new file mode 100644 index 000000000..8fbd1aa5d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLFunctionWithUnknownResult.java @@ -0,0 +1,174 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public enum YSQLFunctionWithUnknownResult { + + ABBREV("abbrev", YSQLDataType.TEXT, YSQLDataType.INET), + BROADCAST("broadcast", YSQLDataType.INET, YSQLDataType.INET), FAMILY("family", YSQLDataType.INT, YSQLDataType.INET), + HOSTMASK("hostmask", YSQLDataType.INET, YSQLDataType.INET), MASKLEN("masklen", YSQLDataType.INT, YSQLDataType.INET), + NETMASK("netmask", YSQLDataType.INET, YSQLDataType.INET), + SET_MASKLEN("set_masklen", YSQLDataType.INET, YSQLDataType.INET, YSQLDataType.INT), + TEXT("text", YSQLDataType.TEXT, YSQLDataType.INET), + INET_SAME_FAMILY("inet_same_family", YSQLDataType.BOOLEAN, YSQLDataType.INET, YSQLDataType.INET), + + // https://www.postgres.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-SIGNAL-TABLE + // PG_RELOAD_CONF("pg_reload_conf", YSQLDataType.BOOLEAN), // too much output + // PG_ROTATE_LOGFILE("pg_rotate_logfile", YSQLDataType.BOOLEAN), prints warning + + // https://www.postgresql.org/docs/devel/functions-info.html#FUNCTIONS-INFO-SESSION-TABLE + CURRENT_DATABASE("current_database", YSQLDataType.TEXT), // name + // CURRENT_QUERY("current_query", YSQLDataType.TEXT), // can generate false positives + CURRENT_SCHEMA("current_schema", YSQLDataType.TEXT), // name + // CURRENT_SCHEMAS("current_schemas", YSQLDataType.TEXT, YSQLDataType.BOOLEAN), + INET_CLIENT_PORT("inet_client_port", YSQLDataType.INT), INET_SERVER_PORT("inet_server_port", YSQLDataType.INT), + PG_BACKEND_PID("pg_backend_pid", YSQLDataType.INT), PG_CURRENT_LOGFILE("pg_current_logfile", YSQLDataType.TEXT), + // PG_IS_OTHER_TEMP_SCHEMA("pg_is_other_temp_schema", YSQLDataType.BOOLEAN), + // PG_JIT_AVAILABLE("pg_is_other_temp_schema", YSQLDataType.BOOLEAN), + PG_NOTIFICATION_QUEUE_USAGE("pg_notification_queue_usage", YSQLDataType.REAL), + PG_TRIGGER_DEPTH("pg_trigger_depth", YSQLDataType.INT), VERSION("version", YSQLDataType.TEXT), + + // + TO_CHAR("to_char", YSQLDataType.TEXT, YSQLDataType.BYTEA, YSQLDataType.TEXT) { + @Override + public YSQLExpression[] getArguments(YSQLDataType returnType, YSQLExpressionGenerator gen, int depth) { + YSQLExpression[] args = super.getArguments(returnType, gen, depth); + args[0] = gen.generateExpression(YSQLDataType.getRandomType()); + return args; + } + }, + + // String functions + ASCII("ascii", YSQLDataType.INT, YSQLDataType.TEXT), + BTRIM("btrim", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + CHR("chr", YSQLDataType.TEXT, YSQLDataType.INT), + CONVERT_FROM("convert_from", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT) { + @Override + public YSQLExpression[] getArguments(YSQLDataType returnType, YSQLExpressionGenerator gen, int depth) { + YSQLExpression[] args = super.getArguments(returnType, gen, depth); + args[1] = YSQLConstant.createTextConstant("UTF8"); + return args; + } + }, + // concat + // segfault + BIT_LENGTH("bit_length", YSQLDataType.INT, YSQLDataType.BYTEA), + INITCAP("initcap", YSQLDataType.TEXT, YSQLDataType.TEXT), + LEFT("left", YSQLDataType.TEXT, YSQLDataType.INT, YSQLDataType.TEXT), + LOWER("lower", YSQLDataType.TEXT, YSQLDataType.TEXT), MD5("md5", YSQLDataType.TEXT, YSQLDataType.TEXT), + UPPER("upper", YSQLDataType.TEXT, YSQLDataType.TEXT), + // PG_CLIENT_ENCODING("pg_client_encoding", YSQLDataType.TEXT), + QUOTE_LITERAL("quote_literal", YSQLDataType.TEXT, YSQLDataType.TEXT), + QUOTE_IDENT("quote_ident", YSQLDataType.TEXT, YSQLDataType.TEXT), + REGEX_REPLACE("regexp_replace", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + // todo mute repeat function because it may provide OOMs + // REPEAT("repeat", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT), + REPLACE("replace", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + REVERSE("reverse", YSQLDataType.TEXT, YSQLDataType.TEXT), + RIGHT("right", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT), + RPAD("rpad", YSQLDataType.TEXT, YSQLDataType.INT, YSQLDataType.TEXT), + RTRIM("rtrim", YSQLDataType.TEXT, YSQLDataType.TEXT), + SPLIT_PART("split_part", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT), + STRPOS("strpos", YSQLDataType.INT, YSQLDataType.TEXT, YSQLDataType.TEXT), + SUBSTR("substr", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.INT, YSQLDataType.INT), + TO_ASCII("to_ascii", YSQLDataType.TEXT, YSQLDataType.TEXT), TO_HEX("to_hex", YSQLDataType.INT, YSQLDataType.TEXT), + TRANSLATE("translate", YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT, YSQLDataType.TEXT), + // mathematical functions + // https://www.postgresql.org/docs/9.5/functions-math.html + ABS("abs", YSQLDataType.REAL, YSQLDataType.REAL), CBRT("cbrt", YSQLDataType.REAL, YSQLDataType.REAL), + CEILING("ceiling", YSQLDataType.REAL), // + DEGREES("degrees", YSQLDataType.REAL), EXP("exp", YSQLDataType.REAL), LN("ln", YSQLDataType.REAL), + LOG("log", YSQLDataType.REAL), LOG2("log", YSQLDataType.REAL, YSQLDataType.REAL), PI("pi", YSQLDataType.REAL), + POWER("power", YSQLDataType.REAL, YSQLDataType.REAL), TRUNC("trunc", YSQLDataType.REAL, YSQLDataType.INT), + TRUNC2("trunc", YSQLDataType.REAL, YSQLDataType.INT, YSQLDataType.REAL), FLOOR("floor", YSQLDataType.REAL), + + // trigonometric functions - complete + // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-TRIG-TABLE + ACOS("acos", YSQLDataType.REAL), // + ACOSD("acosd", YSQLDataType.REAL), // + ASIN("asin", YSQLDataType.REAL), // + ASIND("asind", YSQLDataType.REAL), // + ATAN("atan", YSQLDataType.REAL), // + ATAND("atand", YSQLDataType.REAL), // + ATAN2("atan2", YSQLDataType.REAL, YSQLDataType.REAL), // + ATAN2D("atan2d", YSQLDataType.REAL, YSQLDataType.REAL), // + COS("cos", YSQLDataType.REAL), // + COSD("cosd", YSQLDataType.REAL), // + COT("cot", YSQLDataType.REAL), // + COTD("cotd", YSQLDataType.REAL), // + SIN("sin", YSQLDataType.REAL), // + SIND("sind", YSQLDataType.REAL), // + TAN("tan", YSQLDataType.REAL), // + TAND("tand", YSQLDataType.REAL), // + + // hyperbolic functions - complete + // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-HYP-TABLE + SINH("sinh", YSQLDataType.REAL), // + COSH("cosh", YSQLDataType.REAL), // + TANH("tanh", YSQLDataType.REAL), // + ASINH("asinh", YSQLDataType.REAL), // + ACOSH("acosh", YSQLDataType.REAL), // + ATANH("atanh", YSQLDataType.REAL), // + + // https://www.postgresql.org/docs/devel/functions-binarystring.html + GET_BIT("get_bit", YSQLDataType.INT, YSQLDataType.TEXT, YSQLDataType.INT), + GET_BYTE("get_byte", YSQLDataType.INT, YSQLDataType.TEXT, YSQLDataType.INT), + + // range functions + // https://www.postgresql.org/docs/devel/functions-range.html#RANGE-FUNCTIONS-TABLE + RANGE_LOWER("lower", YSQLDataType.INT, YSQLDataType.RANGE), // + RANGE_UPPER("upper", YSQLDataType.INT, YSQLDataType.RANGE), // + RANGE_ISEMPTY("isempty", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_LOWER_INC("lower_inc", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_UPPER_INC("upper_inc", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_LOWER_INF("lower_inf", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_UPPER_INF("upper_inf", YSQLDataType.BOOLEAN, YSQLDataType.RANGE), // + RANGE_MERGE("range_merge", YSQLDataType.RANGE, YSQLDataType.RANGE, YSQLDataType.RANGE), // + + // https://www.postgresql.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE + GET_COLUMN_SIZE("get_column_size", YSQLDataType.INT, YSQLDataType.TEXT); + // PG_DATABASE_SIZE("pg_database_size", YSQLDataType.INT, YSQLDataType.INT); + // PG_SIZE_BYTES("pg_size_bytes", YSQLDataType.INT, YSQLDataType.TEXT); + + private final String functionName; + private final YSQLDataType returnType; + private final YSQLDataType[] argTypes; + + YSQLFunctionWithUnknownResult(String functionName, YSQLDataType returnType, YSQLDataType... indexType) { + this.functionName = functionName; + this.returnType = returnType; + this.argTypes = indexType.clone(); + } + + public static List getSupportedFunctions(YSQLDataType type) { + List functions = new ArrayList<>(); + for (YSQLFunctionWithUnknownResult func : values()) { + if (func.isCompatibleWithReturnType(type)) { + functions.add(func); + } + } + return functions; + } + + public boolean isCompatibleWithReturnType(YSQLDataType t) { + return t == returnType; + } + + public YSQLExpression[] getArguments(YSQLDataType returnType, YSQLExpressionGenerator gen, int depth) { + YSQLExpression[] args = new YSQLExpression[argTypes.length]; + for (int i = 0; i < args.length; i++) { + args[i] = gen.generateExpression(depth, argTypes[i]); + } + return args; + + } + + public String getName() { + return functionName; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java new file mode 100644 index 000000000..a8c5b9490 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLInOperation.java @@ -0,0 +1,65 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.List; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLInOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final List listElements; + private final boolean isTrue; + + public YSQLInOperation(YSQLExpression expr, List listElements, boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public YSQLExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + public boolean isTrue() { + return isTrue; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant leftValue = expr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return YSQLConstant.createNullConstant(); + } + boolean isNull = false; + for (YSQLExpression expr : getListElements()) { + YSQLConstant rightExpectedValue = expr.getExpectedValue(); + if (rightExpectedValue == null) { + return null; + } + if (rightExpectedValue.isNull()) { + isNull = true; + } else if (rightExpectedValue.isEquals(this.expr.getExpectedValue()).isBoolean() + && rightExpectedValue.isEquals(this.expr.getExpectedValue()).asBoolean()) { + return YSQLConstant.createBooleanConstant(isTrue); + } + } + + if (isNull) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createBooleanConstant(!isTrue); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java new file mode 100644 index 000000000..746d46cfd --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java @@ -0,0 +1,49 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLJoin implements YSQLExpression { + + private final YSQLExpression tableReference; + private final YSQLExpression onClause; + private final YSQLJoinType type; + + public YSQLJoin(YSQLExpression tableReference, YSQLExpression onClause, YSQLJoinType type) { + this.tableReference = tableReference; + this.onClause = onClause; + this.type = type; + } + + public YSQLExpression getTableReference() { + return tableReference; + } + + public YSQLExpression getOnClause() { + return onClause; + } + + public YSQLJoinType getType() { + return type; + } + + @Override + public YSQLDataType getExpressionType() { + throw new AssertionError(); + } + + @Override + public YSQLConstant getExpectedValue() { + throw new AssertionError(); + } + + public enum YSQLJoinType { + INNER, LEFT, RIGHT, FULL, CROSS; + + public static YSQLJoinType getRandom() { + return Randomly.fromOptions(values()); + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java b/src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java new file mode 100644 index 000000000..e57a347f9 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLOrderByTerm.java @@ -0,0 +1,42 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLOrderByTerm implements YSQLExpression { + + private final YSQLOrder order; + private final YSQLExpression expr; + + public YSQLOrderByTerm(YSQLExpression expr, YSQLOrder order) { + this.expr = expr; + this.order = order; + } + + public YSQLOrder getOrder() { + return order; + } + + public YSQLExpression getExpr() { + return expr; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + + @Override + public YSQLConstant getExpectedValue() { + throw new AssertionError(this); + } + + public enum YSQLOrder { + ASC, DESC; + + public static YSQLOrder getRandomOrder() { + return Randomly.fromOptions(YSQLOrder.values()); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java new file mode 100644 index 000000000..975ac6aaf --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPOSIXRegularExpression.java @@ -0,0 +1,65 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPOSIXRegularExpression implements YSQLExpression { + + private final YSQLExpression string; + private final YSQLExpression regex; + private final POSIXRegex op; + + public YSQLPOSIXRegularExpression(YSQLExpression string, YSQLExpression regex, POSIXRegex op) { + this.string = string; + this.regex = regex; + this.op = op; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + return null; + } + + public YSQLExpression getRegex() { + return regex; + } + + public YSQLExpression getString() { + return string; + } + + public POSIXRegex getOp() { + return op; + } + + public enum POSIXRegex implements Operator { + MATCH_CASE_SENSITIVE("~"), MATCH_CASE_INSENSITIVE("~*"), NOT_MATCH_CASE_SENSITIVE("!~"), + NOT_MATCH_CASE_INSENSITIVE("!~*"); + + private final String repr; + + POSIXRegex(String repr) { + this.repr = repr; + } + + public static POSIXRegex getRandom() { + return Randomly.fromOptions(values()); + } + + public String getStringRepresentation() { + return repr; + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java new file mode 100644 index 000000000..65ab056c5 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixOperation.java @@ -0,0 +1,146 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPostfixOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final PostfixOperator op; + private final String operatorTextRepresentation; + + public YSQLPostfixOperation(YSQLExpression expr, PostfixOperator op) { + this.expr = expr; + this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); + this.op = op; + } + + public static YSQLExpression create(YSQLExpression expr, PostfixOperator op) { + return new YSQLPostfixOperation(expr, op); + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.apply(expectedValue); + } + + public String getOperatorTextRepresentation() { + return operatorTextRepresentation; + } + + public YSQLExpression getExpression() { + return expr; + } + + public enum PostfixOperator implements Operator { + IS_NULL("IS NULL", "ISNULL") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return YSQLDataType.values(); + } + + }, + IS_UNKNOWN("IS UNKNOWN") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + }, + + IS_NOT_NULL("IS NOT NULL", "NOTNULL") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return YSQLDataType.values(); + } + + }, + IS_NOT_UNKNOWN("IS NOT UNKNOWN") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + return YSQLConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + }, + IS_TRUE("IS TRUE") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + return YSQLConstant.createFalse(); + } else { + return YSQLConstant.createBooleanConstant(expectedValue.cast(YSQLDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + + }, + IS_FALSE("IS FALSE") { + @Override + public YSQLConstant apply(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + return YSQLConstant.createFalse(); + } else { + return YSQLConstant.createBooleanConstant(!expectedValue.cast(YSQLDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public YSQLDataType[] getInputDataTypes() { + return new YSQLDataType[] { YSQLDataType.BOOLEAN }; + } + + }; + + private final String[] textRepresentations; + + PostfixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public static PostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract YSQLConstant apply(YSQLConstant expectedValue); + + public abstract YSQLDataType[] getInputDataTypes(); + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java new file mode 100644 index 000000000..af9e64498 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPostfixText.java @@ -0,0 +1,36 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPostfixText implements YSQLExpression { + + private final YSQLExpression expr; + private final String text; + private final YSQLConstant expectedValue; + private final YSQLDataType type; + + public YSQLPostfixText(YSQLExpression expr, String text, YSQLConstant expectedValue, YSQLDataType type) { + this.expr = expr; + this.text = text; + this.expectedValue = expectedValue; + this.type = type; + } + + public YSQLExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + @Override + public YSQLDataType getExpressionType() { + return type; + } + + @Override + public YSQLConstant getExpectedValue() { + return expectedValue; + } +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java b/src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java new file mode 100644 index 000000000..d34f07567 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLPrefixOperation.java @@ -0,0 +1,115 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.IgnoreMeException; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLPrefixOperation implements YSQLExpression { + + private final YSQLExpression expr; + private final PrefixOperator op; + + public YSQLPrefixOperation(YSQLExpression expr, PrefixOperator op) { + this.expr = expr; + this.op = op; + } + + @Override + public YSQLDataType getExpressionType() { + return op.getExpressionType(); + } + + @Override + public YSQLConstant getExpectedValue() { + YSQLConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.getExpectedValue(expectedValue); + } + + public YSQLDataType[] getInputDataTypes() { + return op.dataTypes; + } + + public String getTextRepresentation() { + return op.textRepresentation; + } + + public YSQLExpression getExpression() { + return expr; + } + + public enum PrefixOperator implements Operator { + NOT("NOT", YSQLDataType.BOOLEAN) { + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + protected YSQLConstant getExpectedValue(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + return YSQLConstant.createNullConstant(); + } else { + return YSQLConstant.createBooleanConstant(!expectedValue.cast(YSQLDataType.BOOLEAN).asBoolean()); + } + } + }, + UNARY_PLUS("+", YSQLDataType.INT) { + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + @Override + protected YSQLConstant getExpectedValue(YSQLConstant expectedValue) { + // TODO: actual converts to double precision + return expectedValue; + } + + }, + UNARY_MINUS("-", YSQLDataType.INT) { + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.INT; + } + + @Override + protected YSQLConstant getExpectedValue(YSQLConstant expectedValue) { + if (expectedValue.isNull()) { + // TODO + throw new IgnoreMeException(); + } + if (expectedValue.isInt() && expectedValue.asInt() == Long.MIN_VALUE) { + throw new IgnoreMeException(); + } + try { + return YSQLConstant.createIntConstant(-expectedValue.asInt()); + } catch (UnsupportedOperationException e) { + return null; + } + } + + }; + + private final String textRepresentation; + private final YSQLDataType[] dataTypes; + + PrefixOperator(String textRepresentation, YSQLDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract YSQLDataType getExpressionType(); + + protected abstract YSQLConstant getExpectedValue(YSQLConstant expectedValue); + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java new file mode 100644 index 000000000..ce477c238 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java @@ -0,0 +1,135 @@ +package sqlancer.yugabyte.ysql.ast; + +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public class YSQLSelect extends SelectBase implements YSQLExpression { + + private SelectType selectOption = SelectType.ALL; + private List joinClauses = Collections.emptyList(); + private YSQLExpression distinctOnClause; + private ForClause forClause; + + public void setSelectType(SelectType fromOptions) { + this.setSelectOption(fromOptions); + } + + public SelectType getSelectOption() { + return selectOption; + } + + public void setSelectOption(SelectType fromOptions) { + this.selectOption = fromOptions; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + + public List getJoinClauses() { + return joinClauses; + } + + public void setJoinClauses(List joinStatements) { + this.joinClauses = joinStatements; + + } + + public YSQLExpression getDistinctOnClause() { + return distinctOnClause; + } + + public void setDistinctOnClause(YSQLExpression distinctOnClause) { + if (selectOption != SelectType.DISTINCT) { + throw new IllegalArgumentException(); + } + this.distinctOnClause = distinctOnClause; + } + + public ForClause getForClause() { + return forClause; + } + + public void setForClause(ForClause forClause) { + this.forClause = forClause; + } + + public enum ForClause { + UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); + + private final String textRepresentation; + + ForClause(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static ForClause getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + + public enum SelectType { + DISTINCT, ALL; + + public static SelectType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public static class YSQLFromTable implements YSQLExpression { + private final YSQLTable t; + private final boolean only; + + public YSQLFromTable(YSQLTable t, boolean only) { + this.t = t; + this.only = only; + } + + public YSQLTable getTable() { + return t; + } + + public boolean isOnly() { + return only; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + } + + public static class YSQLSubquery implements YSQLExpression { + private final YSQLSelect s; + private final String name; + + public YSQLSubquery(YSQLSelect s, String name) { + this.s = s; + this.name = name; + } + + public YSQLSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public YSQLDataType getExpressionType() { + return null; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java b/src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java new file mode 100644 index 000000000..794525bc2 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLSimilarTo.java @@ -0,0 +1,39 @@ +package sqlancer.yugabyte.ysql.ast; + +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; + +public class YSQLSimilarTo implements YSQLExpression { + + private final YSQLExpression string; + private final YSQLExpression similarTo; + private final YSQLExpression escapeCharacter; + + public YSQLSimilarTo(YSQLExpression string, YSQLExpression similarTo, YSQLExpression escapeCharacter) { + this.string = string; + this.similarTo = similarTo; + this.escapeCharacter = escapeCharacter; + } + + public YSQLExpression getString() { + return string; + } + + public YSQLExpression getSimilarTo() { + return similarTo; + } + + public YSQLExpression getEscapeCharacter() { + return escapeCharacter; + } + + @Override + public YSQLDataType getExpressionType() { + return YSQLDataType.BOOLEAN; + } + + @Override + public YSQLConstant getExpectedValue() { + return null; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java new file mode 100644 index 000000000..035b2fbe4 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java @@ -0,0 +1,182 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public class YSQLAlterTableGenerator { + + private final YSQLTable randomTable; + private final Randomly r; + private final YSQLGlobalState globalState; + + public YSQLAlterTableGenerator(YSQLTable randomTable, YSQLGlobalState globalState) { + this.randomTable = randomTable; + this.globalState = globalState; + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter create(YSQLTable randomTable, YSQLGlobalState globalState) { + return new YSQLAlterTableGenerator(randomTable, globalState).generate(); + } + + public List getActions(ExpectedErrors errors) { + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonInsertUpdateErrors(errors); + YSQLCommon.addCommonTableErrors(errors); + errors.add("duplicate key value violates unique constraint"); + errors.add("cannot drop key column"); + errors.add("cannot drop desired object(s) because other objects depend on them"); + errors.add("invalid input syntax for"); + errors.add("cannot remove a key column"); + errors.add("it has pending trigger events"); + errors.add("could not open relation"); + errors.add("functions in index expression must be marked IMMUTABLE"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("has no default operator class for access method"); + errors.add("does not accept data type"); + errors.add("does not exist for access method"); + errors.add("could not find cast from"); + errors.add("does not exist"); // TODO: investigate + errors.add("constraints on permanent tables may reference only permanent tables"); + List action; + if (Randomly.getBoolean()) { + action = Randomly.nonEmptySubset(Action.values()); + } else { + // make it more likely that the ALTER TABLE succeeds + action = Randomly.subset(Randomly.smallNumber(), Action.values()); + } + if (randomTable.getColumns().size() == 1) { + action.remove(Action.ALTER_TABLE_DROP_COLUMN); + } + if (!randomTable.hasIndexes()) { + action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); + } + if (action.isEmpty()) { + throw new IgnoreMeException(); + } + return action; + } + + public SQLQueryAdapter generate() { + ExpectedErrors errors = new ExpectedErrors(); + int i = 0; + List action = getActions(errors); + StringBuilder sb = new StringBuilder(); + sb.append("ALTER TABLE "); + if (Randomly.getBoolean()) { + sb.append(" ONLY"); + errors.add("cannot use ONLY for foreign key on partitioned table"); + } + sb.append(" "); + sb.append(randomTable.getName()); + sb.append(" "); + for (Action a : action) { + if (i++ != 0) { + sb.append(", "); + } + switch (a) { + case ALTER_TABLE_DROP_COLUMN: + sb.append("DROP "); + if (Randomly.getBoolean()) { + sb.append(" IF EXISTS "); + } + sb.append(randomTable.getRandomColumn().getName()); + errors.add("because other objects depend on it"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("RESTRICT", "CASCADE")); + } + errors.add("does not exist"); + errors.add("cannot drop column"); + errors.add("cannot drop key column"); + errors.add("cannot drop inherited column"); + break; + case ADD_TABLE_CONSTRAINT: + sb.append("ADD "); + sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); + YSQLCommon.addTableConstraint(sb, randomTable, globalState, errors); + errors.add("already exists"); + errors.add("multiple primary keys for table"); + errors.add("could not create unique index"); + errors.add("contains null values"); + errors.add("cannot cast type"); + errors.add("unsupported PRIMARY KEY constraint with partition key definition"); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("insufficient columns in UNIQUE constraint definition"); + errors.add("which is part of the partition key"); + errors.add("out of range"); + errors.add("there is no unique constraint matching given keys for referenced table"); + errors.add("constraints on temporary tables may reference only temporary tables"); + errors.add("constraints on unlogged tables may reference only permanent or unlogged tables"); + errors.add("constraints on permanent tables may reference only permanent tables"); + errors.add("cannot reference partitioned table"); + errors.add("cannot be implemented"); + errors.add("violates foreign key constraint"); + errors.add("unsupported ON COMMIT and foreign key combination"); + errors.add("USING INDEX is not supported on partitioned tables"); + if (Randomly.getBoolean()) { + sb.append(" NOT VALID"); + errors.add("cannot be marked NOT VALID"); + errors.add("cannot add NOT VALID foreign key on partitioned table"); + } else { + errors.add("is violated by some row"); + } + break; + case ADD_TABLE_CONSTRAINT_USING_INDEX: + sb.append("ADD "); + sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); + sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); + sb.append(" USING INDEX "); + sb.append(randomTable.getRandomIndex().getIndexName()); + errors.add("already exists"); + errors.add("PRIMARY KEY containing column of type"); + errors.add("not valid"); + errors.add("is not a unique index"); + errors.add("is already associated with a constraint"); + errors.add("Cannot create a primary key or unique constraint using such an index"); + errors.add("multiple primary keys for table"); + errors.add("appears twice in unique constraint"); + errors.add("appears twice in primary key constraint"); + errors.add("contains null values"); + errors.add("insufficient columns in PRIMARY KEY constraint definition"); + errors.add("which is part of the partition key"); + break; + case DISABLE_ROW_LEVEL_SECURITY: + sb.append("DISABLE ROW LEVEL SECURITY"); + break; + case ENABLE_ROW_LEVEL_SECURITY: + sb.append("ENABLE ROW LEVEL SECURITY"); + break; + case FORCE_ROW_LEVEL_SECURITY: + sb.append("FORCE ROW LEVEL SECURITY"); + break; + case NO_FORCE_ROW_LEVEL_SECURITY: + sb.append("NO FORCE ROW LEVEL SECURITY"); + break; + default: + throw new AssertionError(a); + } + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + protected enum Action { + // ALTER_TABLE_ADD_COLUMN, // [ COLUMN ] column data_type [ COLLATE collation ] [ + // column_constraint [ ... ] ] + ALTER_TABLE_DROP_COLUMN, // DROP [ COLUMN ] [ IF EXISTS ] column [ RESTRICT | CASCADE ] + ADD_TABLE_CONSTRAINT, // ADD table_constraint [ NOT VALID ] + ADD_TABLE_CONSTRAINT_USING_INDEX, // ADD table_constraint_using_index + DISABLE_ROW_LEVEL_SECURITY, // DISABLE ROW LEVEL SECURITY + ENABLE_ROW_LEVEL_SECURITY, // ENABLE ROW LEVEL SECURITY + FORCE_ROW_LEVEL_SECURITY, // FORCE ROW LEVEL SECURITY + NO_FORCE_ROW_LEVEL_SECURITY, // NO FORCE ROW LEVEL SECURITY + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java new file mode 100644 index 000000000..bee2ec5ef --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAnalyzeGenerator.java @@ -0,0 +1,37 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public final class YSQLAnalyzeGenerator { + + private YSQLAnalyzeGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + YSQLTable table = globalState.getSchema().getRandomTable(); + StringBuilder sb = new StringBuilder("ANALYZE"); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(" VERBOSE"); + sb.append(")"); + } + sb.append(" "); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(table.getRandomNonEmptyColumnSubset().stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + } + + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("deadlock")); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java new file mode 100644 index 000000000..6e852d167 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLClusterGenerator.java @@ -0,0 +1,32 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public final class YSQLClusterGenerator { + + private YSQLClusterGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("there is no previously clustered index for table"); + errors.add("cannot cluster a partitioned table"); + errors.add("access method does not support clustering"); + StringBuilder sb = new StringBuilder("CLUSTER "); + if (Randomly.getBoolean()) { + YSQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean() && !table.getIndexes().isEmpty()) { + sb.append(" USING "); + sb.append(table.getRandomIndex().getIndexName()); + errors.add("cannot cluster on partial index"); + } + } + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java new file mode 100644 index 000000000..f020a8994 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommentGenerator.java @@ -0,0 +1,68 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +/** + * @see COMMENT + */ +public final class YSQLCommentGenerator { + + private YSQLCommentGenerator() { + } + + public static SQLQueryAdapter generate(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("COMMENT ON "); + Action type = Randomly.fromOptions(Action.values()); + YSQLTable randomTable = globalState.getSchema().getRandomTable(); + switch (type) { + case INDEX: + sb.append("INDEX "); + if (randomTable.getIndexes().isEmpty()) { + throw new IgnoreMeException(); + } else { + sb.append(randomTable.getRandomIndex().getIndexName()); + } + break; + case COLUMN: + sb.append("COLUMN "); + sb.append(randomTable.getRandomColumn().getFullQualifiedName()); + break; + case STATISTICS: + sb.append("STATISTICS "); + if (randomTable.getStatistics().isEmpty()) { + throw new IgnoreMeException(); + } else { + sb.append(randomTable.getStatistics().get(0).getName()); + } + break; + case TABLE: + sb.append("TABLE "); + if (randomTable.isView()) { + throw new IgnoreMeException(); + } + sb.append(randomTable.getName()); + break; + default: + throw new AssertionError(type); + } + sb.append(" IS "); + if (Randomly.getBoolean()) { + sb.append("NULL"); + } else { + sb.append("'"); + sb.append(globalState.getRandomly().getString().replace("'", "''")); + sb.append("'"); + } + return new SQLQueryAdapter(sb.toString()); + } + + private enum Action { + INDEX, COLUMN, STATISTICS, TABLE + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java new file mode 100644 index 000000000..e92af3e0f --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java @@ -0,0 +1,441 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; + +public final class YSQLCommon { + + private YSQLCommon() { + } + + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.add("An I/O error occurred while sending to the backend"); + errors.add("Conflicts with committed transaction"); + errors.add("cannot be changed"); + errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); + + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); + errors.add("but it cannot be referenced from this part of the query"); + errors.add("missing FROM-clause entry for table"); + + errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); + } + + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.add("PRIMARY KEY containing column of type 'INET' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'VARBIT' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'INT4RANGE' not yet supported"); + errors.add("INDEX on column of type 'INET' not yet supported"); + errors.add("INDEX on column of type 'VARBIT' not yet supported"); + errors.add("INDEX on column of type 'INT4RANGE' not yet supported"); + errors.add("is not commutative"); // exclude + errors.add("cannot be changed"); + errors.add("operator requires run-time type coercion"); // exclude + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.add("syntax error at or near \"(\""); + errors.add("does not exist"); + errors.add("is not unique"); + errors.add("cannot be changed"); + errors.add("invalid reference to FROM-clause entry for table"); + + errors.add("Invalid column number"); + errors.add("specified more than once"); + errors.add("You might need to add explicit type casts"); + errors.add("invalid regular expression"); + errors.add("could not determine which collation to use"); + errors.add("invalid input syntax for integer"); + errors.add("invalid regular expression"); + errors.add("operator does not exist"); + errors.add("quantifier operand invalid"); + errors.add("collation mismatch"); + errors.add("collations are not supported"); + errors.add("operator is not unique"); + errors.add("is not a valid binary digit"); + errors.add("invalid hexadecimal digit"); + errors.add("invalid hexadecimal data: odd number of digits"); + errors.add("zero raised to a negative power is undefined"); + errors.add("cannot convert infinity to numeric"); + errors.add("division by zero"); + errors.add("invalid input syntax for type money"); + errors.add("invalid input syntax for type"); + errors.add("cannot cast type"); + errors.add("value overflows numeric format"); + errors.add("is of type boolean but expression is of type text"); + errors.add("a negative number raised to a non-integer power yields a complex result"); + errors.add("could not determine polymorphic type because input has type unknown"); + addToCharFunctionErrors(errors); + addBitStringOperationErrors(errors); + addFunctionErrors(errors); + addCommonRangeExpressionErrors(errors); + addCommonRegexExpressionErrors(errors); + } + + private static void addToCharFunctionErrors(ExpectedErrors errors) { + errors.add("multiple decimal points"); + errors.add("and decimal point together"); + errors.add("multiple decimal points"); + errors.add("cannot use \"S\" twice"); + errors.add("must be ahead of \"PR\""); + errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); + errors.add("cannot use \"S\" and \"SG\" together"); + errors.add("cannot use \"S\" and \"MI\" together"); + errors.add("cannot use \"S\" and \"PL\" together"); + errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); + errors.add("is not a number"); + } + + private static void addBitStringOperationErrors(ExpectedErrors errors) { + errors.add("cannot XOR bit strings of different sizes"); + errors.add("cannot AND bit strings of different sizes"); + errors.add("cannot OR bit strings of different sizes"); + errors.add("must be type boolean, not type text"); + } + + private static void addFunctionErrors(ExpectedErrors errors) { + errors.add("out of valid range"); // get_bit/get_byte + errors.add("cannot take logarithm of a negative number"); + errors.add("cannot take logarithm of zero"); + errors.add("requested character too large for encoding"); // chr + errors.add("null character not permitted"); // chr + errors.add("requested character not valid for encoding"); // chr + errors.add("requested length too large"); // repeat + errors.add("invalid memory alloc request size"); // repeat + errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + } + + private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + errors.add("is not a valid hexadecimal digit"); + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.add("range lower bound must be less than or equal to range upper bound"); + errors.add("result of range difference would not be contiguous"); + errors.add("out of range"); + errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.add("value too long for type character"); + errors.add("not found in view targetlist"); + } + + public static boolean appendDataType(YSQLDataType type, StringBuilder sb, boolean allowSerial, + boolean generateOnlyKnown, List opClasses) throws AssertionError { + boolean serial = false; + switch (type) { + case BOOLEAN: + sb.append("boolean"); + break; + case INT: + if (Randomly.getBoolean() && allowSerial) { + serial = true; + sb.append(Randomly.fromOptions("serial", "bigserial")); + } else { + sb.append(Randomly.fromOptions("smallint", "integer", "bigint")); + } + break; + case TEXT: + if (Randomly.getBoolean()) { + sb.append("TEXT"); + } else if (Randomly.getBoolean()) { + // TODO: support CHAR (without VAR) + if (YSQLProvider.generateOnlyKnown || Randomly.getBoolean()) { + sb.append("VAR"); + } + sb.append("CHAR"); + sb.append("("); + sb.append(ThreadLocalRandom.current().nextInt(1, 500)); + sb.append(")"); + } else { + sb.append("name"); + } + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + case REAL: + if (Randomly.getBoolean()) { + sb.append("REAL"); + } else { + sb.append("FLOAT"); + } + break; + case RANGE: + sb.append(Randomly.fromOptions("int4range", "int4range")); // , "int8range", "numrange" + break; + case MONEY: + sb.append("money"); + break; + case BYTEA: + sb.append("bytea"); + break; + case BIT: + sb.append("BIT"); + // if (Randomly.getBoolean()) { + sb.append(" VARYING"); + // } + sb.append("("); + sb.append(Randomly.getNotCachedInteger(1, 500)); + sb.append(")"); + break; + case INET: + sb.append("inet"); + break; + default: + throw new AssertionError(type); + } + return serial; + } + + public static void generateWith(StringBuilder sb, YSQLGlobalState globalState, ExpectedErrors errors, + List columnsToBeAdded, boolean isTemporaryTable) { + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" "); + // disabled https://github.com/yugabyte/yugabyte-db/issues/11357 + // sb.append(" WITH ("); + // ArrayList values = new ArrayList<>(Arrays.asList(StorageParameters.values())); + // errors.add("unrecognized parameter"); + // errors.add("ALTER TABLE / ADD CONSTRAINT USING INDEX is not supported on partitioned tables"); + // List subset = Randomly.nonEmptySubset(values); + // int i = 0; + // for (StorageParameters parameter : subset) { + // if (i++ != 0) { + // sb.append(", "); + // } + // sb.append(parameter.parameter); + // sb.append("="); + // sb.append(parameter.op.apply(globalState.getRandomly())); + // } + // sb.append(")"); + } else { + sb.append(" WITHOUT OIDS "); + } + } else if (Randomly.getBoolean() && !isTemporaryTable) { + if (Randomly.getBoolean()) { + sb.append(" SPLIT INTO "); + sb.append(Randomly.smallNumber() + 1); + sb.append(" TABLETS "); + + errors.add("cannot create colocated table with split option"); + errors.add("columns must be present to split by number of tablets"); + errors.add("option is not yet supported for hash partitioned tables"); + } else { + sb.append(" SPLIT AT VALUES ("); + + errors.add("cannot create colocated table with split option"); + errors.add("SPLIT AT option is not yet supported for hash partitioned tables"); + errors.add("Cannot have duplicate split rows"); // just in case + + boolean hasBoolean = false; + for (YSQLColumn column : columnsToBeAdded) { + if (column.getType().equals(YSQLDataType.BOOLEAN)) { + hasBoolean = true; + break; + } + } + + int splits = hasBoolean ? 2 : Randomly.smallNumber() + 2; + long start = Randomly.smallNumber(); + boolean[] bools = { false, true }; + for (int i = 1; i <= splits; i++) { + int size = columnsToBeAdded.size(); + int counter = 1; + for (YSQLColumn c : columnsToBeAdded) { + sb.append("("); + switch (c.getType()) { + case INT: + case REAL: + sb.append(YSQLConstant.createDoubleConstant(start)); + case FLOAT: + sb.append(YSQLConstant.createIntConstant(start)); + break; + case BOOLEAN: + sb.append(YSQLConstant.createBooleanConstant(bools[i - 1])); + break; + case TEXT: + sb.append(YSQLConstant.createTextConstant(String.valueOf(start))); + break; + default: + throw new IgnoreMeException(); + } + sb.append(")"); + counter++; + start += Randomly.smallNumber() + 1; + if (counter <= size) { + sb.append(","); + } + } + + if (i < splits) { + sb.append(","); + } + } + sb.append(")"); + } + } else if (Randomly.getBoolean()) { + errors.add("Cannot use TABLEGROUP with TEMP table"); + sb.append(" TABLEGROUP tg").append( + Randomly.getNotCachedInteger(1, (int) YSQLTableGroupGenerator.UNIQUE_TABLEGROUP_COUNTER.get())); + } + } + + public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder sb, YSQLTable table, + YSQLGlobalState globalState, ExpectedErrors errors) { + // TODO constraint name + List tableConstraints = Randomly.nonEmptySubset(TableConstraints.values()); + if (excludePrimaryKey) { + tableConstraints.remove(TableConstraints.PRIMARY_KEY); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + tableConstraints.remove(TableConstraints.FOREIGN_KEY); + } + for (TableConstraints t : tableConstraints) { + sb.append(", "); + // TODO add index parameters + addTableConstraint(sb, table, globalState, t, errors); + } + } + + public static void addTableConstraint(StringBuilder sb, YSQLTable table, YSQLGlobalState globalState, + ExpectedErrors errors) { + addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); + } + + private static void addTableConstraint(StringBuilder sb, YSQLTable table, YSQLGlobalState globalState, + TableConstraints t, ExpectedErrors errors) { + List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); + List otherColumns; + YSQLCommon.addCommonExpressionErrors(errors); + switch (t) { + case CHECK: + sb.append("CHECK("); + sb.append(YSQLVisitor.getExpressionAsString(globalState, YSQLDataType.BOOLEAN, table.getColumns())); + sb.append(")"); + errors.add("constraint must be added to child tables too"); + errors.add("missing FROM-clause entry for table"); + break; + case UNIQUE: + sb.append("UNIQUE("); + sb.append(randomNonEmptyColumnSubset.stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + break; + case PRIMARY_KEY: + sb.append("PRIMARY KEY("); + sb.append(randomNonEmptyColumnSubset.stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + break; + case FOREIGN_KEY: + sb.append("FOREIGN KEY ("); + sb.append(randomNonEmptyColumnSubset.stream().map(AbstractTableColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(") REFERENCES "); + YSQLTable randomOtherTable = globalState.getSchema().getRandomTable(tab -> !tab.isView()); + sb.append(randomOtherTable.getName()); + if (randomOtherTable.getColumns().size() < randomNonEmptyColumnSubset.size()) { + throw new IgnoreMeException(); + } + otherColumns = randomOtherTable.getRandomNonEmptyColumnSubset(randomNonEmptyColumnSubset.size()); + sb.append("("); + sb.append(otherColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("MATCH FULL", "MATCH SIMPLE")); + } + if (Randomly.getBoolean()) { + sb.append(" ON DELETE "); + errors.add("ERROR: invalid ON DELETE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" ON UPDATE "); + errors.add("invalid ON UPDATE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" "); + if (Randomly.getBoolean()) { + sb.append("DEFERRABLE"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("INITIALLY DEFERRED", "INITIALLY IMMEDIATE")); + } + } else { + sb.append("NOT DEFERRABLE"); + } + } + break; + default: + throw new AssertionError(t); + } + } + + private static void deleteOrUpdateAction(StringBuilder sb) { + sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in GROUP BY"); + } + + public static void addViewErrors(ExpectedErrors errors) { + errors.add("already exists"); + errors.add("cannot drop columns from view"); + errors.add("non-integer constant in ORDER BY"); // TODO + errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO + errors.add("cannot change data type of view column"); + errors.add("specified more than once"); // TODO + errors.add("materialized views must not use temporary tables or views"); + errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); + errors.add("is not a view"); + errors.add("non-integer constant in DISTINCT ON"); + errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + } + + public enum TableConstraints { + CHECK, UNIQUE, PRIMARY_KEY, FOREIGN_KEY + } + + // private enum StorageParameters { + // COLOCATED("COLOCATED", (r) -> Randomly.getBoolean()); + // // TODO + // + // private final String parameter; + // private final Function op; + // + // StorageParameters(String parameter, Function op) { + // this.parameter = parameter; + // this.op = op; + // } + // } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java new file mode 100644 index 000000000..e2eeb1ceb --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -0,0 +1,46 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; + +public final class YSQLDeleteGenerator { + + private YSQLDeleteGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + YSQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("violates foreign key constraint"); + errors.add("violates not-null constraint"); + errors.add("could not determine which collation to use for string comparison"); + StringBuilder sb = new StringBuilder("DELETE FROM"); + if (Randomly.getBoolean()) { + sb.append(" ONLY"); + } + sb.append(" "); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(YSQLVisitor.asString( + YSQLExpressionGenerator.generateExpression(globalState, table.getColumns(), YSQLDataType.BOOLEAN))); + } + if (Randomly.getBoolean()) { + sb.append(" RETURNING "); + sb.append( + YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, table.getColumns()))); + } + YSQLCommon.addCommonExpressionErrors(errors); + errors.add("out of range"); + errors.add("cannot cast"); + errors.add("invalid input syntax for"); + errors.add("division by zero"); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java new file mode 100644 index 000000000..40ccdfa0d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java @@ -0,0 +1,39 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable.TableType; + +public final class YSQLDiscardGenerator { + + private YSQLDiscardGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("DISCARD "); + // prevent that DISCARD discards all tables (if they are TEMP tables) + boolean hasNonTempTables = globalState.getSchema().getDatabaseTables().stream() + .anyMatch(t -> t.getTableType() == TableType.STANDARD); + String what; + if (hasNonTempTables) { + what = Randomly.fromOptions("ALL", "PLANS", "SEQUENCES", "TEMPORARY", "TEMP"); + } else { + what = Randomly.fromOptions("PLANS", "SEQUENCES"); + } + sb.append(what); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { + + @Override + public boolean couldAffectSchema() { + return canDiscardTemporaryTables(what); + } + }; + } + + private static boolean canDiscardTemporaryTables(String what) { + return what.contentEquals("TEMPORARY") || what.contentEquals("TEMP") || what.contentEquals("ALL"); + } +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java new file mode 100644 index 000000000..da9b5ae61 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDropIndexGenerator.java @@ -0,0 +1,41 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLIndex; + +public final class YSQLDropIndexGenerator { + + private YSQLDropIndexGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + List indexes = globalState.getSchema().getRandomTable().getIndexes(); + StringBuilder sb = new StringBuilder(); + sb.append("DROP INDEX "); + if (Randomly.getBoolean() || indexes.isEmpty()) { + sb.append("IF EXISTS "); + if (indexes.isEmpty() || Randomly.getBoolean()) { + sb.append(DBMSCommon.createIndexName(Randomly.smallNumber())); + } else { + sb.append(Randomly.fromList(indexes).getIndexName()); + } + } else { + sb.append(Randomly.fromList(indexes).getIndexName()); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot drop desired object(s) because other objects depend on them", + "cannot drop index", "does not exist"), + true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java new file mode 100644 index 000000000..c6a554d67 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java @@ -0,0 +1,563 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLRowValue; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryArithmeticOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryBitOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryComparisonOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryLogicalOperation; +import sqlancer.yugabyte.ysql.ast.YSQLBinaryRangeOperation; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConcatOperation; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLFunction; +import sqlancer.yugabyte.ysql.ast.YSQLFunctionWithUnknownResult; +import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; +import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; + +public class YSQLExpressionGenerator implements ExpressionGenerator { + + private final int maxDepth; + + private final Randomly r; + private final Map functionsAndTypes; + private final List allowedFunctionTypes; + private List columns; + private YSQLRowValue rw; + private boolean expectedResult; + private YSQLGlobalState globalState; + private boolean allowAggregateFunctions; + + public YSQLExpressionGenerator(YSQLGlobalState globalState) { + this.r = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + this.globalState = globalState; + this.functionsAndTypes = globalState.getFunctionsAndTypes(); + this.allowedFunctionTypes = globalState.getAllowedFunctionTypes(); + } + + public static YSQLExpression generateExpression(YSQLGlobalState globalState, YSQLDataType type) { + return new YSQLExpressionGenerator(globalState).generateExpression(0, type); + } + + private static YSQLCompoundDataType getCompoundDataType(YSQLDataType type) { + switch (type) { + case BOOLEAN: + case DECIMAL: // TODO + case FLOAT: + case INT: + case MONEY: + case RANGE: + case REAL: + case INET: + case BYTEA: + return YSQLCompoundDataType.create(type); + case TEXT: // TODO + case BIT: + if (Randomly.getBoolean() + || YSQLProvider.generateOnlyKnown /* + * The PQS implementation does not check for size specifications + */) { + return YSQLCompoundDataType.create(type); + } else { + return YSQLCompoundDataType.create(type, (int) Randomly.getNotCachedInteger(1, 1000)); + } + default: + throw new AssertionError(type); + } + + } + + public static YSQLExpression generateConstant(Randomly r, YSQLDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return YSQLConstant.createNullConstant(); + } + // if (Randomly.getBooleanWithSmallProbability()) { + // return YSQLConstant.createTextConstant(r.getString()); + // } + switch (type) { + case INT: + if (Randomly.getBooleanWithSmallProbability()) { + return YSQLConstant.createTextConstant(String.valueOf(r.getInteger())); + } else { + return YSQLConstant.createIntConstant(r.getInteger()); + } + case BOOLEAN: + if (Randomly.getBooleanWithSmallProbability() && !YSQLProvider.generateOnlyKnown) { + return YSQLConstant + .createTextConstant(Randomly.fromOptions("TR", "TRUE", "FA", "FALSE", "0", "1", "ON", "off")); + } else { + return YSQLConstant.createBooleanConstant(Randomly.getBoolean()); + } + case TEXT: + return YSQLConstant.createTextConstant(r.getString()); + case DECIMAL: + return YSQLConstant.createDecimalConstant(r.getRandomBigDecimal()); + case FLOAT: + return YSQLConstant.createFloatConstant((float) r.getDouble()); + case REAL: + return YSQLConstant.createDoubleConstant(r.getDouble()); + case RANGE: + return YSQLConstant.createRange(r.getInteger(), Randomly.getBoolean(), r.getInteger(), + Randomly.getBoolean()); + case MONEY: + return new YSQLCastOperation(generateConstant(r, YSQLDataType.FLOAT), + getCompoundDataType(YSQLDataType.MONEY)); + case INET: + return YSQLConstant.createInetConstant(getRandomInet(r)); + case BIT: + return YSQLConstant.createBitConstant(r.getInteger()); + case BYTEA: + return YSQLConstant.createByteConstant(String.valueOf(r.getInteger())); + default: + throw new AssertionError(type); + } + } + + private static String getRandomInet(Randomly r) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 4; i++) { + if (i != 0) { + sb.append('.'); + } + sb.append(r.getInteger() & 255); + } + return sb.toString(); + } + + public static YSQLExpression generateExpression(YSQLGlobalState globalState, List columns, + YSQLDataType type) { + return new YSQLExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); + } + + public static YSQLExpression generateExpression(YSQLGlobalState globalState, List columns) { + return new YSQLExpressionGenerator(globalState).setColumns(columns).generateExpression(0); + + } + + public YSQLExpressionGenerator setColumns(List columns) { + this.columns = columns; + return this; + } + + public YSQLExpressionGenerator setRowValue(YSQLRowValue rw) { + this.rw = rw; + return this; + } + + public YSQLExpression generateExpression(int depth) { + return generateExpression(depth, YSQLDataType.getRandomType()); + } + + public List generateOrderBy() { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new YSQLOrderByTerm(YSQLColumnValue.create(Randomly.fromList(columns), null), + YSQLOrderByTerm.YSQLOrder.getRandomOrder())); + } + return orderBys; + } + + private YSQLExpression generateFunctionWithUnknownResult(int depth, YSQLDataType type) { + List supportedFunctions = YSQLFunctionWithUnknownResult + .getSupportedFunctions(type); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + supportedFunctions = supportedFunctions.stream() + .filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (supportedFunctions.isEmpty()) { + throw new IgnoreMeException(); + } + YSQLFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); + return new YSQLFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); + } + + private YSQLExpression generateFunctionWithKnownResult(int depth, YSQLDataType type) { + List functions = Stream.of(YSQLFunction.YSQLFunctionWithResult.values()) + .filter(f -> f.supportsReturnType(type)).collect(Collectors.toList()); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (functions.isEmpty()) { + throw new IgnoreMeException(); + } + YSQLFunction.YSQLFunctionWithResult randomFunction = Randomly.fromList(functions); + int nrArgs = randomFunction.getNrArgs(); + if (randomFunction.isVariadic()) { + nrArgs += Randomly.smallNumber(); + } + YSQLDataType[] argTypes = randomFunction.getInputTypesForReturnType(type, nrArgs); + YSQLExpression[] args = new YSQLExpression[nrArgs]; + do { + for (int i = 0; i < args.length; i++) { + args[i] = generateExpression(depth + 1, argTypes[i]); + } + } while (!randomFunction.checkArguments(args)); + return new YSQLFunction(randomFunction, type, args); + } + + private YSQLExpression generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (YSQLProvider.generateOnlyKnown) { + validOptions.remove(BooleanExpression.SIMILAR_TO); + validOptions.remove(BooleanExpression.POSIX_REGEX); + validOptions.remove(BooleanExpression.BINARY_RANGE_COMPARISON); + } + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + YSQLPostfixOperation.PostfixOperator random = YSQLPostfixOperation.PostfixOperator.getRandom(); + return YSQLPostfixOperation + .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); + case IN_OPERATION: + return inOperation(depth + 1); + case NOT: + return new YSQLPrefixOperation(generateExpression(depth + 1, YSQLDataType.BOOLEAN), + YSQLPrefixOperation.PrefixOperator.NOT); + case BINARY_LOGICAL_OPERATOR: + YSQLExpression first = generateExpression(depth + 1, YSQLDataType.BOOLEAN); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + first = new YSQLBinaryLogicalOperation(first, generateExpression(depth + 1, YSQLDataType.BOOLEAN), + YSQLBinaryLogicalOperation.BinaryLogicalOperator.getRandom()); + } + return first; + case BINARY_COMPARISON: + YSQLDataType dataType = getMeaningfulType(); + return generateComparison(depth, dataType); + case CAST: + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(YSQLDataType.BOOLEAN)); + case FUNCTION: + return generateFunction(depth + 1, YSQLDataType.BOOLEAN); + case BETWEEN: + YSQLDataType type = getMeaningfulType(); + return new YSQLBetweenOperation(generateExpression(depth + 1, type), generateExpression(depth + 1, type), + generateExpression(depth + 1, type), Randomly.getBoolean()); + case SIMILAR_TO: + assert !expectedResult; + // TODO also generate the escape character + return new YSQLSimilarTo(generateExpression(depth + 1, YSQLDataType.TEXT), + generateExpression(depth + 1, YSQLDataType.TEXT), null); + case POSIX_REGEX: + assert !expectedResult; + return new YSQLPOSIXRegularExpression(generateExpression(depth + 1, YSQLDataType.TEXT), + generateExpression(depth + 1, YSQLDataType.TEXT), + YSQLPOSIXRegularExpression.POSIXRegex.getRandom()); + case BINARY_RANGE_COMPARISON: + // TODO element check + return new YSQLBinaryRangeOperation(YSQLBinaryRangeOperation.YSQLBinaryRangeComparisonOperator.getRandom(), + generateExpression(depth + 1, YSQLDataType.RANGE), + generateExpression(depth + 1, YSQLDataType.RANGE)); + default: + throw new AssertionError(); + } + } + + private YSQLDataType getMeaningfulType() { + // make it more likely that the expression does not only consist of constant + // expressions + if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { + return YSQLDataType.getRandomType(); + } else { + return Randomly.fromList(columns).getType(); + } + } + + private YSQLExpression generateFunction(int depth, YSQLDataType type) { + if (YSQLProvider.generateOnlyKnown || Randomly.getBoolean()) { + return generateFunctionWithKnownResult(depth, type); + } else { + return generateFunctionWithUnknownResult(depth, type); + } + } + + private YSQLExpression generateComparison(int depth, YSQLDataType dataType) { + YSQLExpression leftExpr = generateExpression(depth + 1, dataType); + YSQLExpression rightExpr = generateExpression(depth + 1, dataType); + return getComparison(leftExpr, rightExpr); + } + + private YSQLExpression getComparison(YSQLExpression leftExpr, YSQLExpression rightExpr) { + return new YSQLBinaryComparisonOperation(leftExpr, rightExpr, + YSQLBinaryComparisonOperation.YSQLBinaryComparisonOperator.getRandom()); + } + + private YSQLExpression inOperation(int depth) { + YSQLDataType type = YSQLDataType.getRandomType(); + YSQLExpression leftExpr = generateExpression(depth + 1, type); + List rightExpr = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + rightExpr.add(generateExpression(depth + 1, type)); + } + return new YSQLInOperation(leftExpr, rightExpr, Randomly.getBoolean()); + } + + public YSQLExpression generateExpression(int depth, YSQLDataType originalType) { + YSQLDataType dataType = originalType; + if (dataType == YSQLDataType.REAL && Randomly.getBoolean()) { + dataType = Randomly.fromOptions(YSQLDataType.INT, YSQLDataType.FLOAT); + } + if (dataType == YSQLDataType.FLOAT && Randomly.getBoolean()) { + dataType = YSQLDataType.INT; + } + return generateExpressionInternal(depth, dataType); + } + + private YSQLExpression generateExpressionInternal(int depth, YSQLDataType dataType) throws AssertionError { + if (allowAggregateFunctions && Randomly.getBoolean()) { + allowAggregateFunctions = false; // aggregate function calls cannot be nested + return getAggregate(dataType); + } + if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { + // generic expression + if (Randomly.getBoolean() || depth > maxDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(r, dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(r, dataType); + } else { + return createColumnOfType(dataType); + } + } + } else { + if (Randomly.getBoolean()) { + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(dataType)); + } else { + return generateFunctionWithUnknownResult(depth, dataType); + } + } + } else { + switch (dataType) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case TEXT: + return generateTextExpression(depth); + case DECIMAL: + case REAL: + case FLOAT: + case MONEY: + case INET: + return generateConstant(r, dataType); + case BYTEA: + return generateByteExpression(); + case BIT: + return generateBitExpression(depth); + case RANGE: + return generateRangeExpression(depth); + default: + throw new AssertionError(dataType); + } + } + } + + private YSQLExpression generateRangeExpression(int depth) { + RangeExpression option; + List validOptions = new ArrayList<>(Arrays.asList(RangeExpression.values())); + option = Randomly.fromList(validOptions); + switch (option) { + case BINARY_OP: + return new YSQLBinaryRangeOperation(YSQLBinaryRangeOperation.YSQLBinaryRangeOperator.getRandom(), + generateExpression(depth + 1, YSQLDataType.RANGE), + generateExpression(depth + 1, YSQLDataType.RANGE)); + default: + throw new AssertionError(option); + } + } + + private YSQLExpression generateTextExpression(int depth) { + TextExpression option; + List validOptions = new ArrayList<>(Arrays.asList(TextExpression.values())); + option = Randomly.fromList(validOptions); + + switch (option) { + case CAST: + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(YSQLDataType.TEXT)); + case FUNCTION: + return generateFunction(depth + 1, YSQLDataType.TEXT); + case CONCAT: + return generateConcat(depth); + default: + throw new AssertionError(); + } + } + + private YSQLExpression generateConcat(int depth) { + YSQLExpression left = generateExpression(depth + 1, YSQLDataType.TEXT); + YSQLExpression right = generateExpression(depth + 1); + return new YSQLConcatOperation(left, right); + } + + private YSQLExpression generateByteExpression() { + return YSQLConstant.createByteConstant("Th\\000omas"); + } + + private YSQLExpression generateBitExpression(int depth) { + BitExpression option; + option = Randomly.fromOptions(BitExpression.values()); + switch (option) { + case BINARY_OPERATION: + return new YSQLBinaryBitOperation(YSQLBinaryBitOperation.YSQLBinaryBitOperator.getRandom(), + generateExpression(depth + 1, YSQLDataType.BIT), generateExpression(depth + 1, YSQLDataType.BIT)); + default: + throw new AssertionError(); + } + } + + private YSQLExpression generateIntExpression(int depth) { + IntExpression option; + option = Randomly.fromOptions(IntExpression.values()); + switch (option) { + case CAST: + return new YSQLCastOperation(generateExpression(depth + 1), getCompoundDataType(YSQLDataType.INT)); + case UNARY_OPERATION: + YSQLExpression intExpression = generateExpression(depth + 1, YSQLDataType.INT); + return new YSQLPrefixOperation(intExpression, Randomly.getBoolean() + ? YSQLPrefixOperation.PrefixOperator.UNARY_PLUS : YSQLPrefixOperation.PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, YSQLDataType.INT); + case BINARY_ARITHMETIC_EXPRESSION: + return new YSQLBinaryArithmeticOperation(generateExpression(depth + 1, YSQLDataType.INT), + generateExpression(depth + 1, YSQLDataType.INT), + YSQLBinaryArithmeticOperation.YSQLBinaryOperator.getRandom()); + default: + throw new AssertionError(); + } + } + + private YSQLExpression createColumnOfType(YSQLDataType type) { + List columns = filterColumns(type); + YSQLColumn fromList = Randomly.fromList(columns); + YSQLConstant value = rw == null ? null : rw.getValues().get(fromList); + return YSQLColumnValue.create(fromList, value); + } + + final List filterColumns(YSQLDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + public YSQLExpression generateExpressionWithExpectedResult(YSQLDataType type) { + this.expectedResult = true; + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns).setRowValue(rw); + YSQLExpression expr; + do { + expr = gen.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + public List generateExpressions(int nr) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(0)); + } + return expressions; + } + + public YSQLExpression generateExpression(YSQLDataType dataType) { + return generateExpression(0, dataType); + } + + public YSQLExpressionGenerator setGlobalState(YSQLGlobalState globalState) { + this.globalState = globalState; + return this; + } + + public YSQLExpression generateHavingClause() { + this.allowAggregateFunctions = true; + YSQLExpression expression = generateExpression(YSQLDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + + public YSQLExpression generateAggregate() { + return getAggregate(YSQLDataType.getRandomType()); + } + + private YSQLExpression getAggregate(YSQLDataType dataType) { + List aggregates = YSQLAggregate.YSQLAggregateFunction + .getAggregates(dataType); + YSQLAggregate.YSQLAggregateFunction agg = Randomly.fromList(aggregates); + return generateArgsForAggregate(dataType, agg); + } + + public YSQLAggregate generateArgsForAggregate(YSQLDataType dataType, YSQLAggregate.YSQLAggregateFunction agg) { + List types = agg.getTypes(dataType); + List args = new ArrayList<>(); + for (YSQLDataType argType : types) { + args.add(generateExpression(argType)); + } + return new YSQLAggregate(args, agg); + } + + public YSQLExpressionGenerator allowAggregates(boolean value) { + allowAggregateFunctions = value; + return this; + } + + @Override + public YSQLExpression generatePredicate() { + return generateExpression(YSQLDataType.BOOLEAN); + } + + @Override + public YSQLExpression negatePredicate(YSQLExpression predicate) { + return new YSQLPrefixOperation(predicate, YSQLPrefixOperation.PrefixOperator.NOT); + } + + @Override + public YSQLExpression isNull(YSQLExpression expr) { + return new YSQLPostfixOperation(expr, YSQLPostfixOperation.PostfixOperator.IS_NULL); + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, CAST, BETWEEN, IN_OPERATION, + SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON + } + + private enum RangeExpression { + BINARY_OP + } + + private enum TextExpression { + CAST, FUNCTION, CONCAT + } + + private enum BitExpression { + BINARY_OPERATION + } + + private enum IntExpression { + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java new file mode 100644 index 000000000..faa7908b0 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -0,0 +1,153 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLIndex; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public final class YSQLIndexGenerator { + + private YSQLIndexGenerator() { + } + + public static SQLQueryAdapter generate(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE"); + if (Randomly.getBoolean()) { + sb.append(" UNIQUE"); + } + sb.append(" INDEX "); + /* + * Commented out as a workaround for https://www.postgresql.org/message-id/CA%2Bu7OA4XYhc- + * qyCgJqwwgMGZDWAyeH821oa5oMzm_HEifZ4BeA%40mail.gmail.com + */ + // if (Randomly.getBoolean()) { + // sb.append("CONCURRENTLY "); + // } + YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized + // views + String indexName = getNewIndexName(randomTable); + sb.append(indexName); + sb.append(" ON "); + if (Randomly.getBoolean()) { + sb.append("ONLY "); + } + sb.append(randomTable.getName()); + IndexType method; + if (Randomly.getBoolean()) { + sb.append(" USING "); + method = Randomly.fromOptions(IndexType.values()); + sb.append(method); + } else { + method = IndexType.BTREE; + } + + sb.append("("); + if (method == IndexType.HASH) { + sb.append(randomTable.getRandomColumn().getName()); + } else { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + if (Randomly.getBoolean()) { + sb.append(randomTable.getRandomColumn().getName()); + } else { + sb.append("("); + YSQLExpression expression = YSQLExpressionGenerator.generateExpression(globalState, + randomTable.getColumns()); + sb.append(YSQLVisitor.asString(expression)); + sb.append(")"); + } + + // if (Randomly.getBoolean()) { + // sb.append(" "); + // sb.append("COLLATE "); + // sb.append(Randomly.fromOptions("C", "POSIX")); + // } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" "); + sb.append(globalState.getRandomOpclass()); + errors.add("does not accept"); + errors.add("does not exist for access method"); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + } + + sb.append(")"); + if (Randomly.getBoolean() && method != IndexType.HASH) { + sb.append(" INCLUDE("); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + } + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + YSQLExpression expr = new YSQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()) + .setGlobalState(globalState).generateExpression(YSQLDataType.BOOLEAN); + sb.append(YSQLVisitor.asString(expr)); + } + errors.add("already contains data"); // CONCURRENT INDEX failed + errors.add("You might need to add explicit type casts"); + errors.add("INDEX on column of type"); + errors.add("collations are not supported"); // TODO check + errors.add("because it has pending trigger events"); + errors.add("duplicate key value violates unique constraint"); + errors.add("could not determine which collation to use for"); + errors.add("index method \"gist\" not supported yet"); + errors.add("is duplicated"); + errors.add("already exists"); + errors.add("could not create unique index"); + errors.add("has no default operator class"); + errors.add("does not support"); + errors.add("cannot cast"); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("insufficient columns in UNIQUE constraint definition"); + errors.add("invalid input syntax for"); + errors.add("must be type "); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("out of range"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("functions in index expression must be marked IMMUTABLE"); + errors.add("result of range difference would not be contiguous"); + errors.add("which is part of the partition key"); + YSQLCommon.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private static String getNewIndexName(YSQLTable randomTable) { + List indexes = randomTable.getIndexes(); + int indexI = 0; + while (true) { + String indexName = DBMSCommon.createIndexName(indexI++); + if (indexes.stream().noneMatch(i -> i.getIndexName().equals(indexName))) { + return indexName; + } + } + } + + public enum IndexType { + BTREE, HASH, GIST, GIN + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java new file mode 100644 index 000000000..1817891b3 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java @@ -0,0 +1,127 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public final class YSQLInsertGenerator { + + private YSQLInsertGenerator() { + } + + public static SQLQueryAdapter insert(YSQLGlobalState globalState) { + YSQLTable table = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("cannot insert into column"); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonInsertUpdateErrors(errors); + YSQLCommon.addCommonExpressionErrors(errors); + errors.add("multiple assignments to same column"); + errors.add("violates foreign key constraint"); + errors.add("value too long for type character varying"); + errors.add("conflicting key value violates exclusion constraint"); + errors.add("violates not-null constraint"); + errors.add("current transaction is aborted"); + errors.add("bit string too long"); + errors.add("new row violates check option for view"); + errors.add("reached maximum value of sequence"); + errors.add("but expression is of type"); + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" OVERRIDING"); + sb.append(" "); + sb.append(Randomly.fromOptions("SYSTEM", "USER")); + sb.append(" VALUE"); + } + sb.append(" VALUES"); + + if (globalState.getDbmsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { + StringBuilder sbRowValue = new StringBuilder(); + sbRowValue.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sbRowValue.append(", "); + } + sbRowValue.append(YSQLVisitor.asString( + YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), columns.get(i).getType()))); + } + sbRowValue.append(")"); + + int n = (int) Randomly.getNotCachedInteger(100, 1000); + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(sbRowValue); + } + } else { + int n = Randomly.smallNumber() + 1; + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + insertRow(globalState, sb, columns, n == 1); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" ON CONFLICT "); + if (Randomly.getBoolean()) { + sb.append("("); + sb.append(table.getRandomColumn().getName()); + sb.append(")"); + errors.add("there is no unique or exclusion constraint matching the ON CONFLICT specification"); + } + sb.append(" DO NOTHING"); + } + errors.add("duplicate key value violates unique constraint"); + errors.add("identity column defined as GENERATED ALWAYS"); + errors.add("out of range"); + errors.add("violates check constraint"); + errors.add("no partition of relation"); + errors.add("invalid input syntax"); + errors.add("division by zero"); + errors.add("violates foreign key constraint"); + errors.add("data type unknown"); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private static void insertRow(YSQLGlobalState globalState, StringBuilder sb, List columns, + boolean canBeDefault) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + if (!Randomly.getBooleanWithSmallProbability() || !canBeDefault) { + YSQLExpression generateConstant; + if (Randomly.getBoolean()) { + generateConstant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), + columns.get(i).getType()); + } else { + generateConstant = new YSQLExpressionGenerator(globalState) + .generateExpression(columns.get(i).getType()); + } + sb.append(YSQLVisitor.asString(generateConstant)); + } else { + sb.append("DEFAULT"); + } + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java new file mode 100644 index 000000000..a6fb58d8e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLNotifyGenerator.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLNotifyGenerator { + + private YSQLNotifyGenerator() { + } + + private static String getChannel() { + return Randomly.fromOptions("asdf", "test"); + } + + public static SQLQueryAdapter createNotify(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("NOTIFY "); + sb.append(getChannel()); + if (Randomly.getBoolean()) { + sb.append(", "); + sb.append("'"); + sb.append(globalState.getRandomly().getString().replace("'", "''")); + sb.append("'"); + } + return new SQLQueryAdapter(sb.toString()); + } + + public static SQLQueryAdapter createListen() { + String sb = "LISTEN " + getChannel(); + return new SQLQueryAdapter(sb); + } + + public static SQLQueryAdapter createUnlisten() { + StringBuilder sb = new StringBuilder(); + sb.append("UNLISTEN "); + if (Randomly.getBoolean()) { + sb.append(getChannel()); + } else { + sb.append("*"); + } + return new SQLQueryAdapter(sb.toString()); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java new file mode 100644 index 000000000..03109227d --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java @@ -0,0 +1,62 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.ForClause; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.SelectType; +import sqlancer.yugabyte.ysql.ast.YSQLSelect.YSQLFromTable; + +public final class YSQLRandomQueryGenerator { + + private YSQLRandomQueryGenerator() { + } + + public static YSQLSelect createRandomQuery(int nrColumns, YSQLGlobalState globalState) { + List columns = new ArrayList<>(); + YSQLTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < nrColumns; i++) { + columns.add(gen.generateExpression(0)); + } + YSQLSelect select = new YSQLSelect(); + select.setSelectType(SelectType.getRandom()); + if (select.getSelectOption() == SelectType.DISTINCT && Randomly.getBoolean()) { + select.setDistinctOnClause(gen.generateExpression(0)); + } + select.setFromList(tables.getTables().stream().map(t -> new YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return select; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java new file mode 100644 index 000000000..7526d8dc6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLReindexGenerator.java @@ -0,0 +1,58 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLIndex; + +public final class YSQLReindexGenerator { + + private YSQLReindexGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("could not create unique index"); // CONCURRENT INDEX + StringBuilder sb = new StringBuilder(); + sb.append("REINDEX"); + // if (Randomly.getBoolean()) { + // sb.append(" VERBOSE"); + // } + sb.append(" "); + Scope scope = Randomly.fromOptions(Scope.values()); + switch (scope) { + case INDEX: + sb.append("INDEX "); + List indexes = globalState.getSchema().getRandomTable().getIndexes(); + if (indexes.isEmpty()) { + throw new IgnoreMeException(); + } + sb.append(indexes.stream().map(YSQLIndex::getIndexName).collect(Collectors.joining())); + break; + case TABLE: + sb.append("TABLE "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + break; + case DATABASE: + sb.append("DATABASE "); + sb.append(globalState.getSchema().getDatabaseName()); + break; + default: + throw new AssertionError(scope); + } + errors.add("already contains data"); // FIXME bug report + errors.add("does not exist"); // internal index + errors.add("REINDEX is not yet implemented for partitioned indexes"); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private enum Scope { + INDEX, TABLE, DATABASE + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java new file mode 100644 index 000000000..3d7104b5e --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLSequenceGenerator.java @@ -0,0 +1,90 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLSequenceGenerator { + + private YSQLSequenceGenerator() { + } + + public static SQLQueryAdapter createSequence(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("TEMPORARY", "TEMP")); + } + sb.append(" SEQUENCE"); + // TODO keep track of sequences + sb.append(" IF NOT EXISTS"); + // TODO generate sequence names + sb.append(" seq"); + if (Randomly.getBoolean()) { + sb.append(" AS "); + sb.append(Randomly.fromOptions("smallint", "integer", "bigint")); + } + if (Randomly.getBoolean()) { + sb.append(" INCREMENT"); + if (Randomly.getBoolean()) { + sb.append(" BY"); + } + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + errors.add("INCREMENT must not be zero"); + } + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" MINVALUE"); + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + } else { + sb.append(" NO MINVALUE"); + } + errors.add("must be less than MAXVALUE"); + } + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" MAXVALUE"); + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + } else { + sb.append(" NO MAXVALUE"); + } + errors.add("must be less than MAXVALUE"); + } + if (Randomly.getBoolean()) { + sb.append(" START"); + if (Randomly.getBoolean()) { + sb.append(" WITH"); + } + sb.append(" "); + sb.append(globalState.getRandomly().getInteger()); + errors.add("cannot be less than MINVALUE"); + errors.add("cannot be greater than MAXVALUE"); + } + if (Randomly.getBoolean()) { + sb.append(" CACHE "); + sb.append(globalState.getRandomly().getPositiveIntegerNotNull()); + } + errors.add("is out of range"); + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" NO"); + } + sb.append(" CYCLE"); + } + if (Randomly.getBoolean()) { + sb.append(" OWNED BY "); + // if (Randomly.getBoolean()) { + sb.append("NONE"); + // } else { + // sb.append(s.getRandomTable().getRandomColumn().getFullQualifiedName()); + // } + } + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java new file mode 100644 index 000000000..3406ca3de --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLSetGenerator.java @@ -0,0 +1,196 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.function.Function; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLSetGenerator { + + private YSQLSetGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + ArrayList options = new ArrayList<>(Arrays.asList(ConfigurationOption.values())); + options.remove(ConfigurationOption.DEFAULT_WITH_OIDS); + ConfigurationOption option = Randomly.fromList(options); + sb.append("SET "); + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions("SESSION", "LOCAL")); + sb.append(" "); + } + sb.append(option.getOptionName()); + sb.append("="); + if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append(option.op.apply(globalState.getRandomly())); + } + // todo avoiding props that are not represented in YSQL + ExpectedErrors errors = new ExpectedErrors(); + errors.add("unrecognized configuration parameter"); + errors.add("cannot be changed"); + + return new SQLQueryAdapter(sb.toString(), errors); + } + + private enum ConfigurationOption { + // YUGABYTE + YB_DEBUG_REPORT_ERROR_STACKTRACE("yb_debug_report_error_stacktrace", + (r) -> Randomly.fromOptions("false", "true")), + YB_DEBUG_LOG_CATCACHE_EVENTS("yb_debug_log_catcache_events", (r) -> Randomly.fromOptions("false", "true")), + YB_DEBUG_LOG_INTERNAL_RESTARTS("yb_debug_log_internal_restarts", (r) -> Randomly.fromOptions("false", "true")), + YB_DEBUG_LOG_DOCDB_REQUESTS("yb_debug_log_docdb_requests", (r) -> Randomly.fromOptions("false", "true")), + // YB_READ_FROM_FOLLOWERS("yb_read_from_followers", (r) -> Randomly.fromOptions("false", "true")), + YB_NON_DDL_TXN_FOR_SYS_TABLES_ALLOWED("yb_non_ddl_txn_for_sys_tables_allowed", + (r) -> Randomly.fromOptions("false", "true")), + YB_TRANSACTION_PRIORITY("yb_transaction_priority", + (r) -> Randomly.fromOptions(0, 0.1, 0.2, 0.3, 0.4, 1, 0.9, 0.8, 0.7, 0.6)), + YB_TRANSACTION_PRIORITY_LOWER_BOUND("yb_transaction_priority_lower_bound", + (r) -> Randomly.fromOptions(0, 0.1, 0.2, 0.3, 0.4)), + YB_TRANSACTION_PRIORITY_UPPER_BOUND("yb_transaction_priority_upper_bound", + (r) -> Randomly.fromOptions(1, 0.9, 0.8, 0.7, 0.6)), + YB_FORMAT_FUNCS_INCLUDE_YB_METADATA("yb_format_funcs_include_yb_metadata", + (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_GEOLOCATION_COSTING("yb_enable_geolocation_costing", (r) -> Randomly.fromOptions("false", "true")), + YB_BINARY_RESTORE("yb_binary_restore", (r) -> Randomly.fromOptions("false", "true")), + YB_TEST_SYSTEM_CATALOGS_CREATION("yb_test_system_catalogs_creation", + (r) -> Randomly.fromOptions("false", "true")), + YB_TEST_FAIL_NEXT_DDL("yb_test_fail_next_ddl", (r) -> Randomly.fromOptions("false", "true")), + YB_DISABLE_TRANSACTIONAL_WRITES("yb_disable_transactional_writes", + (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_OPTIMIZER_STATISTICS("yb_enable_optimizer_statistics", (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_EXPRESSION_PUSHDOWN("yb_enable_expression_pushdown", (r) -> Randomly.fromOptions("false", "true")), + YB_ENABLE_UPSERT_MODE("yb_enable_upsert_mode", (r) -> Randomly.fromOptions("false", "true")), + YB_PLANNER_CUSTOM_PLAN_FOR_PARTITION_PRUNING("yb_planner_custom_plan_for_partition_pruning", + (r) -> Randomly.fromOptions("false", "true")), + YB_INDEX_STATE_FLAGS_UPDATE_DELAY("yb_index_state_flags_update_delay", + (r) -> Randomly.getNotCachedInteger(200, 1000)), + YB_TEST_PLANNER_CUSTOM_PLAN_THRESHOLD("yb_test_planner_custom_plan_threshold", + (r) -> Randomly.getNotCachedInteger(1, Integer.MAX_VALUE)), + // YSQL values + YSQL_UPGRADE_MODE("ysql_upgrade_mode", (r) -> Randomly.fromOptions("false", "true")), + YSQL_SESSION_MAX_BATCH_SIZE("ysql_session_max_batch_size", + (r) -> Randomly.getNotCachedInteger(1, Integer.MAX_VALUE)), + YSQL_MAX_IN_FLIGHT_OPS("ysql_max_in_flight_ops", (r) -> Randomly.getNotCachedInteger(1, Integer.MAX_VALUE)), + // https://www.postgresql.org/docs/11/runtime-config-wal.html + // This parameter can only be set at server start. + // WAL_LEVEL("wal_level", (r) -> Randomly.fromOptions("replica", "minimal", "logical")), + // FSYNC("fsync", (r) -> Randomly.fromOptions(1, 0)), + SYNCHRONOUS_COMMIT("synchronous_commit", + (r) -> Randomly.fromOptions("remote_apply", "remote_write", "local", "off")), + WAL_COMPRESSION("wal_compression", (r) -> Randomly.fromOptions(1, 0)), + // wal_buffer: server start + // wal_writer_delay: server start + // wal_writer_flush_after + COMMIT_DELAY("commit_delay", (r) -> r.getInteger(0, 100000)), + COMMIT_SIBLINGS("commit_siblings", (r) -> r.getInteger(0, 1000)), + // 19.5.2. Checkpoints + // checkpoint_timeout + // checkpoint_completion_target + // checkpoint_flush_after + // checkpoint_warning + // max_wal_size + // min_wal_size + // 19.5.3. Archiving + // archive_mode + // archive_command + // archive_timeout + // https://www.postgresql.org/docs/11/runtime-config-statistics.html + // 19.9.1. Query and Index Statistics Collector + TRACK_ACTIVITIES("track_activities", (r) -> Randomly.fromOptions(1, 0)), + // track_activity_query_size + TRACK_COUNTS("track_counts", (r) -> Randomly.fromOptions(1, 0)), + TRACK_IO_TIMING("track_io_timing", (r) -> Randomly.fromOptions(1, 0)), + TRACK_FUNCTIONS("track_functions", (r) -> Randomly.fromOptions("'none'", "'pl'", "'all'")), + // stats_temp_directory + // TODO 19.9.2. Statistics Monitoring + // https://www.postgresql.org/docs/11/runtime-config-autovacuum.html + // all can only be set at server-conf time + // 19.11. Client Connection Defaults + VACUUM_FREEZE_TABLE_AGE("vacuum_freeze_table_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), + VACUUM_FREEZE_MIN_AGE("vacuum_freeze_min_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 1000000000)), + VACUUM_MULTIXACT_FREEZE_TABLE_AGE("vacuum_multixact_freeze_table_age", + (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), + VACUUM_MULTIXACT_FREEZE_MIN_AGE("vacuum_multixact_freeze_min_age", + (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 1000000000)), + VACUUM_CLEANUP_INDEX_SCALE_FACTOR("vacuum_cleanup_index_scale_factor", + (r) -> Randomly.fromOptions(0.0, 0.0000001, 0.00001, 0.01, 0.1, 1, 10, 100, 100000, 10000000000.0)), + // TODO others + GIN_FUZZY_SEARCH_LIMIT("gin_fuzzy_search_limit", (r) -> r.getInteger(0, 2147483647)), + // 19.13. Version and Platform Compatibility + DEFAULT_WITH_OIDS("default_with_oids", (r) -> Randomly.fromOptions(0, 1)), + SYNCHRONIZED_SEQSCANS("synchronize_seqscans", (r) -> Randomly.fromOptions(0, 1)), + // https://www.postgresql.org/docs/devel/runtime-config-query.html + ENABLE_BITMAPSCAN("enable_bitmapscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_GATHERMERGE("enable_gathermerge", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_HASHJOIN("enable_hashjoin", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_INDEXSCAN("enable_indexscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_INDEXONLYSCAN("enable_indexonlyscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_MATERIAL("enable_material", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_MERGEJOIN("enable_mergejoin", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_NESTLOOP("enable_nestloop", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARALLEL_APPEND("enable_parallel_append", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARALLEL_HASH("enable_parallel_hash", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARTITION_PRUNING("enable_partition_pruning", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARTITIONWISE_JOIN("enable_partitionwise_join", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_PARTITIONWISE_AGGREGATE("enable_partitionwise_aggregate", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_SEGSCAN("enable_seqscan", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_SORT("enable_sort", (r) -> Randomly.fromOptions(1, 0)), + ENABLE_TIDSCAN("enable_tidscan", (r) -> Randomly.fromOptions(1, 0)), + // 19.7.2. Planner Cost Constants (complete as of March 2020) + // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-CONSTANTS + SEQ_PAGE_COST("seq_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + RANDOM_PAGE_COST("random_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + CPU_TUPLE_COST("cpu_tuple_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + CPU_INDEX_TUPLE_COST("cpu_index_tuple_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), + CPU_OPERATOR_COST("cpu_operator_cost", (r) -> Randomly.fromOptions(0d, 0.000001, 0.0025, 0.1, 1, 10, 10000)), + PARALLEL_SETUP_COST("parallel_setup_cost", (r) -> r.getLong(0, Long.MAX_VALUE)), + PARALLEL_TUPLE_COST("parallel_tuple_cost", (r) -> r.getLong(0, Long.MAX_VALUE)), + MIN_PARALLEL_TABLE_SCAN_SIZE("min_parallel_table_scan_size", (r) -> r.getInteger(0, 715827882)), + MIN_PARALLEL_INDEX_SCAN_SIZE("min_parallel_index_scan_size", (r) -> r.getInteger(0, 715827882)), + EFFECTIVE_CACHE_SIZE("effective_cache_size", (r) -> r.getInteger(1, 2147483647)), + JIT_ABOVE_COST("jit_above_cost", (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE - 1))), + JIT_INLINE_ABOVE_COST("jit_inline_above_cost", (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE))), + JIT_OPTIMIZE_ABOVE_COST("jit_optimize_above_cost", + (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE))), + // 19.7.3. Genetic Query Optimizer (complete as of March 2020) + // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-GEQO + GEQO("geqo", (r) -> Randomly.fromOptions(1, 0)), + GEQO_THRESHOLD("geqo_threshold", (r) -> r.getInteger(2, 2147483647)), + GEQO_EFFORT("geqo_effort", (r) -> r.getInteger(1, 10)), + GEQO_POO_SIZE("geqo_pool_size", (r) -> r.getInteger(0, 2147483647)), + GEQO_GENERATIONS("geqo_generations", (r) -> r.getInteger(0, 2147483647)), + GEQO_SELECTION_BIAS("geqo_selection_bias", (r) -> Randomly.fromOptions(1.5, 1.8, 2.0)), + GEQO_SEED("geqo_seed", (r) -> Randomly.fromOptions(0, 0.5, 1)), + // 19.7.4. Other Planner Options (complete as of March 2020) + // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-OTHER + DEFAULT_STATISTICS_TARGET("default_statistics_target", (r) -> r.getInteger(1, 10000)), + CONSTRAINT_EXCLUSION("constraint_exclusion", (r) -> Randomly.fromOptions("on", "off", "partition")), + CURSOR_TUPLE_FRACTION("cursor_tuple_fraction", + (r) -> Randomly.fromOptions(0.0, 0.1, 0.000001, 1, 0.5, 0.9999999)), + FROM_COLLAPSE_LIMIT("from_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), + JIT("jit", (r) -> Randomly.fromOptions(1, 0)), + JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), + PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), + FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")); + + private final String optionName; + private final Function op; + + ConfigurationOption(String optionName, Function op) { + this.optionName = optionName; + this.op = op; + } + + public String getOptionName() { + return optionName; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java new file mode 100644 index 000000000..3b212b00b --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLStatisticsGenerator.java @@ -0,0 +1,74 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLStatisticsObject; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; + +public final class YSQLStatisticsGenerator { + + private YSQLStatisticsGenerator() { + } + + public static SQLQueryAdapter insert(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE STATISTICS "); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO materialized view + if (randomTable.getColumns().size() < 2) { + throw new IgnoreMeException(); + } + sb.append(" "); + sb.append(getNewStatisticsName(randomTable)); + if (Randomly.getBoolean()) { + sb.append(" ("); + List statsSubset; + statsSubset = Randomly.nonEmptySubset("ndistinct", "dependencies", "mcv"); + sb.append(String.join(", ", statsSubset)); + sb.append(")"); + } + + List randomColumns = randomTable.getRandomNonEmptyColumnSubset( + globalState.getRandomly().getInteger(2, randomTable.getColumns().size())); + sb.append(" ON "); + sb.append(randomColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(" FROM "); + sb.append(randomTable.getName()); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot have more than 8 columns in statistics"), + true); + } + + public static SQLQueryAdapter remove(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder("DROP STATISTICS "); + YSQLTable randomTable = globalState.getSchema().getRandomTable(); + List statistics = randomTable.getStatistics(); + if (statistics.isEmpty()) { + throw new IgnoreMeException(); + } + sb.append(Randomly.fromList(statistics).getName()); + return new SQLQueryAdapter(sb.toString(), true); + } + + private static String getNewStatisticsName(YSQLTable randomTable) { + List statistics = randomTable.getStatistics(); + int i = 0; + while (true) { + String candidateName = "s" + i; + if (statistics.stream().noneMatch(stat -> stat.getName().contentEquals(candidateName))) { + return candidateName; + } + i++; + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java new file mode 100644 index 000000000..5e376ab4c --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java @@ -0,0 +1,247 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public class YSQLTableGenerator { + + protected final ExpectedErrors errors = new ExpectedErrors(); + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + private final List columnsToBeAdded = new ArrayList<>(); + private final YSQLTable table; + private final boolean generateOnlyKnown; + private final YSQLGlobalState globalState; + private boolean columnCanHavePrimaryKey; + private boolean columnHasPrimaryKey; + private boolean isTemporaryTable; + + public YSQLTableGenerator(String tableName, boolean generateOnlyKnown, YSQLGlobalState globalState) { + this.tableName = tableName; + this.generateOnlyKnown = generateOnlyKnown; + this.globalState = globalState; + table = new YSQLTable(tableName, columnsToBeAdded, null, null, null, false, false); + // YB catalog specific messages + errors.add("The catalog snapshot used for this transaction has been invalidated"); + + errors.add("PRIMARY KEY containing column of type"); + errors.add("specified value cannot be cast to type boolean for column"); + errors.add("already exists"); + errors.add("invalid input syntax for"); + errors.add("is not unique"); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("cannot create partitioned table as inheritance child"); + errors.add("cannot cast"); + errors.add("ERROR: functions in index expression must be marked IMMUTABLE"); + errors.add("functions in partition key expression must be marked IMMUTABLE"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("has no default operator class for access method"); + errors.add("does not exist for access method"); + errors.add("does not accept data type"); + errors.add("but default expression is of type text"); + errors.add("has pseudo-type unknown"); + errors.add("no collation was derived for partition key column"); + errors.add("cannot set colocated true on a non-colocated database"); + errors.add("Cannot split table that does not have primary key"); + errors.add("inherits from generated column but specifies identity"); + errors.add("inherits from generated column but specifies default"); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonTableErrors(errors); + } + + public static SQLQueryAdapter generate(String tableName, boolean generateOnlyKnown, YSQLGlobalState globalState) { + return new YSQLTableGenerator(tableName, generateOnlyKnown, globalState).generate(); + } + + private SQLQueryAdapter generate() { + columnCanHavePrimaryKey = true; + sb.append("CREATE"); + if (Randomly.getBooleanWithSmallProbability()) { + sb.append(" "); + isTemporaryTable = true; + sb.append(Randomly.fromOptions("TEMPORARY", "TEMP")); + } + sb.append(" TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + createStandard(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void createStandard() throws AssertionError { + sb.append("("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + String name = DBMSCommon.createColumnName(i); + createColumn(name); + } + if (Randomly.getBoolean()) { + errors.add("constraints on temporary tables may reference only temporary tables"); + errors.add("constraints on unlogged tables may reference only permanent or unlogged tables"); + errors.add("constraints on permanent tables may reference only permanent tables"); + errors.add("cannot be implemented"); + errors.add("there is no unique constraint matching given keys for referenced table"); + errors.add("cannot reference partitioned table"); + errors.add("unsupported ON COMMIT and foreign key combination"); + errors.add("ERROR: invalid ON DELETE action for foreign key constraint containing generated column"); + errors.add("exclusion constraints are not supported on partitioned tables"); + errors.add("option is not yet supported for hash partitioned tables"); + YSQLCommon.addTableConstraints(columnHasPrimaryKey, sb, table, globalState, errors); + } + sb.append(")"); + generatePartitionBy(); + YSQLCommon.generateWith(sb, globalState, errors, columnsToBeAdded, isTemporaryTable); + if (Randomly.getBoolean() && isTemporaryTable) { + sb.append(" ON COMMIT "); + // todo ON COMMIT DROP fails and it's known issue + // sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS", "DROP")); + sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS")); + sb.append(" "); + } + } + + private void createColumn(String name) throws AssertionError { + sb.append(name); + sb.append(" "); + YSQLDataType type = YSQLDataType.getRandomType(); + boolean serial = YSQLCommon.appendDataType(type, sb, true, generateOnlyKnown, globalState.getCollates()); + YSQLColumn c = new YSQLColumn(name, type); + c.setTable(table); + columnsToBeAdded.add(c); + sb.append(" "); + if (Randomly.getBoolean()) { + createColumnConstraint(type, serial); + } + } + + private void generatePartitionBy() { + if (Randomly.getBoolean()) { + return; + } + sb.append(" PARTITION BY "); + // TODO "RANGE", + String partitionOption = Randomly.fromOptions("RANGE", "LIST", "HASH"); + sb.append(partitionOption); + sb.append("("); + errors.add("unrecognized parameter"); + errors.add("cannot use constant expression"); + errors.add("unrecognized parameter"); + errors.add("unsupported PRIMARY KEY constraint with partition key definition"); + errors.add("which is part of the partition key."); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("does not accept data type"); + int n = partitionOption.contentEquals("LIST") ? 1 : Randomly.smallNumber() + 1; + YSQLCommon.addCommonExpressionErrors(errors); + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("("); + YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, columnsToBeAdded); + sb.append(YSQLVisitor.asString(expr)); + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(globalState.getRandomOpclass()); + errors.add("does not exist for access method"); + } + } + sb.append(")"); + } + + private void createColumnConstraint(YSQLDataType type, boolean serial) { + List constraintSubset = Randomly.nonEmptySubset(ColumnConstraint.values()); + if (Randomly.getBoolean()) { + // make checks constraints less likely + constraintSubset.remove(ColumnConstraint.CHECK); + } + if (!columnCanHavePrimaryKey || columnHasPrimaryKey) { + constraintSubset.remove(ColumnConstraint.PRIMARY_KEY); + } + if (constraintSubset.contains(ColumnConstraint.GENERATED) + && constraintSubset.contains(ColumnConstraint.DEFAULT)) { + // otherwise: ERROR: both default and identity specified for column + constraintSubset.remove(Randomly.fromOptions(ColumnConstraint.GENERATED, ColumnConstraint.DEFAULT)); + } + if (constraintSubset.contains(ColumnConstraint.GENERATED) && type != YSQLDataType.INT) { + // otherwise: ERROR: identity column type must be smallint, integer, or bigint + constraintSubset.remove(ColumnConstraint.GENERATED); + } + if (serial) { + constraintSubset.remove(ColumnConstraint.GENERATED); + constraintSubset.remove(ColumnConstraint.DEFAULT); + constraintSubset.remove(ColumnConstraint.NULL_OR_NOT_NULL); + + } + for (ColumnConstraint c : constraintSubset) { + sb.append(" "); + switch (c) { + case NULL_OR_NOT_NULL: + sb.append(Randomly.fromOptions("NOT NULL", "NULL")); + errors.add("conflicting NULL/NOT NULL declarations"); + break; + case UNIQUE: + sb.append("UNIQUE"); + break; + case PRIMARY_KEY: + sb.append("PRIMARY KEY"); + columnHasPrimaryKey = true; + break; + case DEFAULT: + sb.append("DEFAULT"); + sb.append(" ("); + sb.append(YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, type))); + sb.append(")"); + errors.add("out of range"); + errors.add("is a generated column"); + break; + case CHECK: + sb.append("CHECK ("); + sb.append(YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, columnsToBeAdded, + YSQLDataType.BOOLEAN))); + sb.append(")"); + errors.add("out of range"); + break; + case GENERATED: + sb.append("GENERATED "); + if (Randomly.getBoolean()) { + sb.append(" ALWAYS AS ("); + sb.append(YSQLVisitor + .asString(YSQLExpressionGenerator.generateExpression(globalState, columnsToBeAdded, type))); + sb.append(") STORED"); + errors.add("A generated column cannot reference another generated column."); + errors.add("cannot use generated column in partition key"); + errors.add("generation expression is not immutable"); + errors.add("cannot use column reference in DEFAULT expression"); + } else { + sb.append(Randomly.fromOptions("ALWAYS", "BY DEFAULT")); + sb.append(" AS IDENTITY"); + } + break; + default: + throw new AssertionError(sb); + } + } + } + + private enum ColumnConstraint { + NULL_OR_NOT_NULL, UNIQUE, PRIMARY_KEY, DEFAULT, CHECK, GENERATED + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java new file mode 100644 index 000000000..da1b5c573 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java @@ -0,0 +1,24 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.concurrent.atomic.AtomicLong; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLTableGroupGenerator { + + // TODO rework + public static final AtomicLong UNIQUE_TABLEGROUP_COUNTER = new AtomicLong(1); + + private YSQLTableGroupGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE TABLEGROUP "); + String tableGroupName = "tg" + UNIQUE_TABLEGROUP_COUNTER.incrementAndGet(); + sb.append(tableGroupName); + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java new file mode 100644 index 000000000..67780c016 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTransactionGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public final class YSQLTransactionGenerator { + + private YSQLTransactionGenerator() { + } + + public static SQLQueryAdapter executeBegin() { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("BEGIN"); + if (Randomly.getBoolean()) { + errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); + sb.append(" ISOLATION LEVEL "); + sb.append(Randomly.fromOptions("SERIALIZABLE", "REPEATABLE READ", "READ COMMITTED")); + // if (Randomly.getBoolean()) { + // sb.append(" "); + // sb.append(Randomly.fromOptions("READ WRITE", "READ ONLY")); + // } + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java new file mode 100644 index 000000000..d10bccee6 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java @@ -0,0 +1,41 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTable; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLTruncateGenerator { + + private YSQLTruncateGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + StringBuilder sb = new StringBuilder(); + sb.append("TRUNCATE"); + if (Randomly.getBoolean()) { + sb.append(" TABLE"); + } + // TODO partitions + // if (Randomly.getBoolean()) { + // sb.append(" ONLY"); + // } + sb.append(" "); + sb.append(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty().stream().map(AbstractTable::getName) + .collect(Collectors.joining(", "))); + // if (Randomly.getBoolean()) { + // sb.append(" "); + // sb.append(Randomly.fromOptions("RESTART IDENTITY", "CONTINUE IDENTITY")); + // } + // if (Randomly.getBoolean()) { + // sb.append(" "); + // sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + // } + return new SQLQueryAdapter(sb.toString(), ExpectedErrors + .from("cannot truncate a table referenced in a foreign key constraint", "is not a table")); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java new file mode 100644 index 000000000..fe8e71ada --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -0,0 +1,76 @@ +package sqlancer.yugabyte.ysql.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; + +public final class YSQLUpdateGenerator { + + private YSQLUpdateGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + YSQLTable randomTable = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); + StringBuilder sb = new StringBuilder(); + sb.append("UPDATE "); + sb.append(randomTable.getName()); + sb.append(" SET "); + ExpectedErrors errors = ExpectedErrors.from("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", + " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable"); + errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced + // table multiple times + errors.add("new row violates check option for view"); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + YSQLCommon.addCommonInsertUpdateErrors(errors); + + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + YSQLColumn column = columns.get(i); + sb.append(column.getName()); + sb.append(" = "); + if (!Randomly.getBoolean()) { + YSQLExpression constant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(YSQLVisitor.asString(constant)); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append("("); + YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), + column.getType()); + // caused by casts + sb.append(YSQLVisitor.asString(expr)); + sb.append(")"); + } + } + errors.add("invalid input syntax for "); + errors.add("operator does not exist: text = boolean"); + errors.add("violates check constraint"); + errors.add("could not determine which collation to use for string comparison"); + errors.add("but expression is of type"); + YSQLCommon.addCommonExpressionErrors(errors); + if (!Randomly.getBooleanWithSmallProbability()) { + sb.append(" WHERE "); + YSQLExpression where = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), + YSQLDataType.BOOLEAN); + sb.append(YSQLVisitor.asString(where)); + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java new file mode 100644 index 000000000..f873f7812 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java @@ -0,0 +1,19 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; + +public final class YSQLVacuumGenerator { + + private YSQLVacuumGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + String sb = "VACUUM"; + ExpectedErrors errors = new ExpectedErrors(); + errors.add("VACUUM cannot run inside a transaction block"); + return new SQLQueryAdapter(sb, errors); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java new file mode 100644 index 000000000..a5b57fbd8 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -0,0 +1,73 @@ +package sqlancer.yugabyte.ysql.gen; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; + +public final class YSQLViewGenerator { + + private YSQLViewGenerator() { + } + + public static SQLQueryAdapter create(YSQLGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE"); + // boolean materialized; + // boolean recursive = false; + if (Randomly.getBoolean()) { + sb.append(" MATERIALIZED"); + // materialized = true; + } else { + if (Randomly.getBoolean()) { + sb.append(" OR REPLACE"); + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" TEMP", " TEMPORARY")); + } + // if (Randomly.getBoolean()) { + // sb.append(" RECURSIVE"); + // recursive = true; + // } + // materialized = false; + } + sb.append(" VIEW "); + int i = 0; + String[] name = new String[1]; + while (true) { + name[0] = "v" + i++; + if (globalState.getSchema().getDatabaseTables().stream() + .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { + break; + } + } + sb.append(name[0]); + sb.append("("); + int nrColumns = Randomly.smallNumber() + 1; + for (i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(DBMSCommon.createColumnName(i)); + } + sb.append(")"); + // if (Randomly.getBoolean() && !materialized && !recursive) { + // sb.append(" WITH "); + // sb.append(Randomly.fromOptions("CASCADED", "LOCAL")); + // sb.append(" CHECK OPTION"); + // errors.add("WITH CHECK OPTION is supported only on automatically updatable views"); + // } + sb.append(" AS ("); + YSQLSelect select = YSQLRandomQueryGenerator.createRandomQuery(nrColumns, globalState); + sb.append(YSQLVisitor.asString(select)); + sb.append(")"); + YSQLCommon.addGroupingErrors(errors); + YSQLCommon.addViewErrors(errors); + YSQLCommon.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java new file mode 100644 index 000000000..a873ce505 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java @@ -0,0 +1,94 @@ +package sqlancer.yugabyte.ysql.oracle; + +import static sqlancer.yugabyte.ysql.YSQLProvider.CREATION_LOCK; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; + +public class YSQLCatalog implements TestOracle { + protected final YSQLGlobalState state; + + protected final ExpectedErrors errors = new ExpectedErrors(); + protected final Main.StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + + private final List dmlActions = Arrays.asList(YSQLProvider.Action.INSERT, + YSQLProvider.Action.UPDATE, YSQLProvider.Action.DELETE); + private final List catalogActions = Arrays.asList(YSQLProvider.Action.CREATE_VIEW, + YSQLProvider.Action.CREATE_SEQUENCE, YSQLProvider.Action.ALTER_TABLE, YSQLProvider.Action.SET_CONSTRAINTS, + YSQLProvider.Action.DISCARD, YSQLProvider.Action.DROP_INDEX, YSQLProvider.Action.COMMENT_ON, + YSQLProvider.Action.RESET_ROLE, YSQLProvider.Action.RESET); + private final List diskActions = Arrays.asList(YSQLProvider.Action.TRUNCATE, + YSQLProvider.Action.VACUUM); + + public YSQLCatalog(YSQLGlobalState globalState) { + this.state = globalState; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + } + + private YSQLProvider.Action getRandomAction(List actions) { + return actions.get(state.getRandomly().getInteger(0, actions.size())); + } + + protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { + synchronized (CREATION_LOCK) { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = YSQLTableGenerator.generate(tableName, true, globalState); + globalState.executeStatement(createTable); + globalState.getManager().incrementSelectQueryCount(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + } catch (IgnoreMeException e) { + // do nothing + } + } + } + } + + @Override + public void check() throws Exception { + // create table or evaluate catalog test + int seed = state.getRandomly().getInteger(1, 100); + if (seed > 95) { + createTables(state, 1); + } else { + YSQLProvider.Action randomAction; + + if (seed > 40) { + randomAction = getRandomAction(dmlActions); + } else if (seed > 10) { + randomAction = getRandomAction(catalogActions); + } else { + randomAction = getRandomAction(diskActions); + } + + state.executeStatement(randomAction.getQuery(state)); + } + state.getManager().incrementSelectQueryCount(); + } +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java new file mode 100644 index 000000000..e8f791b3f --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java @@ -0,0 +1,108 @@ +package sqlancer.yugabyte.ysql.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import com.typesafe.config.Config; +import com.typesafe.config.ConfigFactory; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLRandomQueryGenerator; + +public class YSQLFuzzer implements TestOracle { + private final YSQLGlobalState globalState; + private final List testQueries; + private final ExpectedErrors errors = new ExpectedErrors(); + + public YSQLFuzzer(YSQLGlobalState globalState) { + this.globalState = globalState; + + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + YSQLCommon.addGroupingErrors(errors); + YSQLCommon.addViewErrors(errors); + + // remove timeout error from scope + errors.add("canceling statement due to statement timeout"); + + // exclude nemesis exceptions + errors.add("terminating connection due to administrator command"); + errors.add("Java heap space"); + errors.add("Connection refused"); + errors.add("Connection to"); + + // get config from -Dconfig.file="path/to/fuzzer.conf" + testQueries = new ArrayList<>(); + try { + Config config = ConfigFactory.load(); + ArrayList queriesList = (ArrayList) config.getList("queries").unwrapped(); + for (Object configValue : queriesList) { + String type = ((String) ((Map) configValue).get("type")).toUpperCase(Locale.ROOT); + Integer weight = (Integer) ((Map) configValue).get("weight"); + + Query query = type.equalsIgnoreCase("SELECT") ? new SelectQuery() + : new ActionQuery(YSQLProvider.Action.valueOf(type)); + + for (int i = 0; i < weight; i++) { + testQueries.add(query); + } + } + } catch (Exception e) { + // do nothing + } finally { + if (testQueries.isEmpty()) { + System.out.println("No configuration found. Using just random select statements"); + testQueries.add(new SelectQuery()); + testQueries.add(new ActionQuery(YSQLProvider.Action.UPDATE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.DELETE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.INSERT)); + } + } + } + + @Override + public void check() throws Exception { + Query s = testQueries.get(globalState.getRandomly().getInteger(0, testQueries.size())); + globalState.executeStatement(s.getQuery(globalState, errors)); + globalState.getManager().incrementSelectQueryCount(); + } + + private static class Query { + public SQLQueryAdapter getQuery(YSQLGlobalState state, ExpectedErrors errors) throws Exception { + throw new IllegalAccessException("Should be implemented"); + }; + } + + private static class ActionQuery extends Query { + private final YSQLProvider.Action action; + + ActionQuery(YSQLProvider.Action action) { + this.action = action; + } + + @Override + public SQLQueryAdapter getQuery(YSQLGlobalState state, ExpectedErrors errors) throws Exception { + return action.getQuery(state); + } + } + + private static class SelectQuery extends Query { + + @Override + public SQLQueryAdapter getQuery(YSQLGlobalState state, ExpectedErrors errors) throws Exception { + return new SQLQueryAdapter( + YSQLVisitor.asString(YSQLRandomQueryGenerator.createRandomQuery(Randomly.smallNumber() + 1, state)) + + ";", + errors); + } + } +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java new file mode 100644 index 000000000..863b63398 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -0,0 +1,165 @@ +package sqlancer.yugabyte.ysql.oracle; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPBase; + +public class YSQLNoRECOracle extends NoRECBase implements TestOracle { + + private final YSQLSchema s; + + public YSQLNoRECOracle(YSQLGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + } + + public static List getJoinStatements(YSQLGlobalState globalState, List columns, + List tables) { + List joinStatements = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLTable table = Randomly.fromList(tables); + tables.remove(table); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLSelect.YSQLSubquery subquery = YSQLTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public void check() throws SQLException { + YSQLTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + YSQLExpression randomWhereCondition = getRandomWhereCondition(columns); + List tables = randomTables.getTables(); + + List joinStatements = getJoinStatements(state, columns, tables); + List fromTables = tables.stream() + .map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); + int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, + firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + private YSQLExpression getRandomWhereCondition(List columns) { + return new YSQLExpressionGenerator(state).setColumns(columns).generateExpression(YSQLDataType.BOOLEAN); + } + + private int getUnoptimizedQueryCount(List fromTables, YSQLExpression randomWhereCondition, + List joinStatements) throws SQLException { + YSQLSelect select = new YSQLSelect(); + YSQLCastOperation isTrue = new YSQLCastOperation(randomWhereCondition, + YSQLCompoundDataType.create(YSQLDataType.INT)); + YSQLPostfixText asText = new YSQLPostfixText(isTrue, " as count", null, YSQLDataType.INT); + select.setFetchColumns(Collections.singletonList(asText)); + select.setFromList(fromTables); + select.setSelectType(YSQLSelect.SelectType.ALL); + select.setJoinClauses(joinStatements); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + YSQLVisitor.asString(select) + ") as res"; + if (options.logEachSelect()) { + logger.writeCurrent(unoptimizedQueryString); + } + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGet(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getOptimizedQueryCount(List randomTables, List columns, + YSQLExpression randomWhereCondition, List joinStatements) throws SQLException { + YSQLSelect select = new YSQLSelect(); + YSQLColumnValue allColumns = new YSQLColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(Arrays.asList(allColumns)); + select.setFromList(randomTables); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new YSQLExpressionGenerator(state).setColumns(columns).generateOrderBy()); + } + select.setSelectType(YSQLSelect.SelectType.ALL); + select.setJoinClauses(joinStatements); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = YSQLVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..95ac268f4 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java @@ -0,0 +1,147 @@ +package sqlancer.yugabyte.ysql.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLRowValue; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; + +public class YSQLPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + + public YSQLPivotedQuerySynthesisOracle(YSQLGlobalState globalState) throws SQLException { + super(globalState); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + } + + /* + * Prevent name collisions by aliasing the column. + */ + private YSQLColumn getFetchValueAliasedColumn(YSQLColumn c) { + YSQLColumn aliasedColumn = new YSQLColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType()); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + private List generateGroupByClause(List columns, YSQLRowValue rw) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> YSQLColumnValue.create(c, rw.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private YSQLConstant generateLimit() { + if (Randomly.getBoolean()) { + return YSQLConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private YSQLExpression generateOffset() { + if (Randomly.getBoolean()) { + return YSQLConstant.createIntConstant(0); + } else { + return null; + } + } + + private YSQLExpression generateRectifiedExpression(List columns, YSQLRowValue rw) { + YSQLExpression expr = new YSQLExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) + .generateExpressionWithExpectedResult(YSQLDataType.BOOLEAN); + YSQLExpression result; + if (expr.getExpectedValue().isNull()) { + result = YSQLPostfixOperation.create(expr, YSQLPostfixOperation.PostfixOperator.IS_NULL); + } else { + result = YSQLPostfixOperation.create(expr, expr.getExpectedValue().cast(YSQLDataType.BOOLEAN).asBoolean() + ? YSQLPostfixOperation.PostfixOperator.IS_TRUE : YSQLPostfixOperation.PostfixOperator.IS_FALSE); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions + sb.append(query.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (YSQLColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + @Override + public SQLQueryAdapter getRectifiedQuery() throws SQLException { + YSQLTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); + + YSQLSelect selectStatement = new YSQLSelect(); + selectStatement.setSelectType(Randomly.fromOptions(YSQLSelect.SelectType.values())); + List columns = randomFromTables.getColumns(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + + fetchColumns = columns; + selectStatement.setFromList(randomFromTables.getTables().stream() + .map(t -> new YSQLSelect.YSQLFromTable(t, false)).collect(Collectors.toList())); + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new YSQLColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + YSQLExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + YSQLExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + YSQLExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + List orderBy = new YSQLExpressionGenerator(globalState).setColumns(columns).generateOrderBy(); + selectStatement.setOrderByExpressions(orderBy); + return new SQLQueryAdapter(YSQLVisitor.asString(selectStatement)); + } + + @Override + protected String getExpectedValues(YSQLExpression expr) { + return YSQLVisitor.asExpectedValues(expr); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java new file mode 100644 index 000000000..aa6dfd530 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -0,0 +1,194 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.postgresql.util.PSQLException; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate.YSQLAggregateFunction; +import sqlancer.yugabyte.ysql.ast.YSQLAlias; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation.PostfixOperator; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation.PrefixOperator; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; + +public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public YSQLTLPAggregateOracle(YSQLGlobalState state) { + super(state); + YSQLCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() throws SQLException { + YSQLAggregateFunction aggregateFunction = Randomly.fromOptions(YSQLAggregateFunction.MAX, + YSQLAggregateFunction.MIN, YSQLAggregateFunction.SUM, YSQLAggregateFunction.BIT_AND, + YSQLAggregateFunction.BIT_OR, YSQLAggregateFunction.BOOL_AND, YSQLAggregateFunction.BOOL_OR, + YSQLAggregateFunction.COUNT); + YSQLAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), + aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + originalQuery = YSQLVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + String queryFormatString = "-- %s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + String assertionMessage = String.format("the results mismatch!\n%s\n%s", firstQueryString, + secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + private String createMetamorphicUnionQuery(YSQLSelect select, YSQLAggregate aggregate, List from) { + String metamorphicQuery; + YSQLExpression whereClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLExpression negatedClause = new YSQLPrefixOperation(whereClause, PrefixOperator.NOT); + YSQLExpression notNullClause = new YSQLPostfixOperation(whereClause, PostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + YSQLSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); + YSQLSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); + YSQLSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += YSQLVisitor.asString(leftSelect) + " UNION ALL " + YSQLVisitor.asString(middleSelect) + + " UNION ALL " + YSQLVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + } catch (PSQLException e) { + throw new AssertionError(queryString, e); + } + return resultString; + } + + private List mapped(YSQLAggregate aggregate) { + switch (aggregate.getFunction()) { + case SUM: + case COUNT: + case BIT_AND: + case BIT_OR: + case BOOL_AND: + case BOOL_OR: + case MAX: + case MIN: + return aliasArgs(Arrays.asList(aggregate)); + // case AVG: + //// List arg = Arrays.asList(new + // YSQLCast(aggregate.getExpr().get(0), + // YSQLDataType.DECIMAL.get())); + // YSQLAggregate sum = new YSQLAggregate(YSQLAggregateFunction.SUM, + // aggregate.getExpr()); + // YSQLCast count = new YSQLCast( + // new YSQLAggregate(YSQLAggregateFunction.COUNT, aggregate.getExpr()), + // YSQLDataType.DECIMAL.get()); + //// YSQLBinaryArithmeticOperation avg = new + // YSQLBinaryArithmeticOperation(sum, count, + // YSQLBinaryArithmeticOperator.DIV); + // return aliasArgs(Arrays.asList(sum, count)); + default: + throw new AssertionError(aggregate.getFunction()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (YSQLExpression expr : originalAggregateArgs) { + args.add(new YSQLAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(YSQLAggregate aggregate) { + switch (aggregate.getFunction()) { + // case AVG: + // return "SUM(agg0::DECIMAL)/SUM(agg1)::DECIMAL"; + case COUNT: + return YSQLAggregateFunction.SUM + "(agg0)"; + default: + return aggregate.getFunction().toString() + "(agg0)"; + } + } + + private YSQLSelect getSelect(List aggregates, List from, YSQLExpression whereClause, + List joinList) { + YSQLSelect leftSelect = new YSQLSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinClauses(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java new file mode 100644 index 000000000..359f12d92 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -0,0 +1,117 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; +import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; +import sqlancer.yugabyte.ysql.ast.YSQLConstant; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; +import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; + +public class YSQLTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + protected YSQLSchema s; + protected YSQLTables targetTables; + protected YSQLExpressionGenerator gen; + protected YSQLSelect select; + + public YSQLTLPBase(YSQLGlobalState state) { + super(state); + YSQLCommon.addCommonExpressionErrors(errors); + YSQLCommon.addCommonFetchErrors(errors); + } + + public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState, String name, YSQLTables tables) { + List columns = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + YSQLSelect select = new YSQLSelect(); + select.setFromList(tables.getTables().stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(YSQLSelect.ForClause.getRandom()); + } + return new YSQLSelect.YSQLSubquery(select, name); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + List tables = targetTables.getTables(); + List joins = getJoinStatements(state, targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + protected List getJoinStatements(YSQLGlobalState globalState, List columns, + List tables) { + return YSQLNoRECOracle.getJoinStatements(state, columns, tables); + // TODO joins + } + + protected void generateSelectBase(List tables, List joins) { + List tableList = tables.stream() + .map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + gen = new YSQLExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new YSQLSelect(); + select.setFetchColumns(generateFetchColumns()); + select.setFromList(tableList); + select.setWhereClause(null); + select.setJoinClauses(joins); + if (Randomly.getBoolean()) { + select.setForClause(YSQLSelect.ForClause.getRandom()); + } + } + + List generateFetchColumns() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new YSQLColumnValue(YSQLColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(targetTables.getColumns()); + for (YSQLColumn c : targetColumns) { + fetchColumns.add(new YSQLColumnValue(c, null)); + } + return fetchColumns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java new file mode 100644 index 000000000..24410aee5 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java @@ -0,0 +1,66 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.gen.YSQLCommon; + +public class YSQLTLPHavingOracle extends YSQLTLPBase { + + public YSQLTLPHavingOracle(YSQLGlobalState state) { + super(state); + YSQLCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + havingCheck(); + } + + @Override + List generateFetchColumns() { + List expressions = gen.allowAggregates(true).generateExpressions(Randomly.smallNumber() + 1); + gen.allowAggregates(false); + return expressions; + } + + protected void havingCheck() throws SQLException { + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(YSQLDataType.BOOLEAN)); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = YSQLVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + select.setHavingClause(predicate); + String firstQueryString = YSQLVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = YSQLVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = YSQLVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + protected YSQLExpression generatePredicate() { + return gen.generateHavingClause(); + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java new file mode 100644 index 000000000..0af8095d0 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java @@ -0,0 +1,45 @@ +package sqlancer.yugabyte.ysql.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.yugabyte.ysql.YSQLGlobalState; +import sqlancer.yugabyte.ysql.YSQLVisitor; + +public class YSQLTLPWhereOracle extends YSQLTLPBase { + + public YSQLTLPWhereOracle(YSQLGlobalState state) { + super(state); + } + + @Override + public void check() throws SQLException { + super.check(); + whereCheck(); + } + + protected void whereCheck() throws SQLException { + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + String originalQueryString = YSQLVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setOrderByExpressions(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = YSQLVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = YSQLVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = YSQLVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } +} From 120bc079d4a141367500e2cfa28a6806657f5ada Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Tue, 4 Oct 2022 22:27:33 +0800 Subject: [PATCH 188/856] Support latest versions of CockroachDB and TiDB --- src/sqlancer/ComparatorHelper.java | 11 ++-- src/sqlancer/Randomly.java | 6 +- src/sqlancer/SQLProviderAdapter.java | 10 +++- src/sqlancer/cockroachdb/CockroachDBBugs.java | 45 +++++++++++++- .../cockroachdb/CockroachDBErrors.java | 60 +++++++++++++------ .../cockroachdb/CockroachDBOptions.java | 2 +- .../cockroachdb/CockroachDBSchema.java | 17 ++++++ .../cockroachdb/ast/CockroachDBConstant.java | 12 ++-- .../CockroachDBCreateStatisticsGenerator.java | 7 +-- .../gen/CockroachDBIndexGenerator.java | 19 ++++-- ...CockroachDBSetClusterSettingGenerator.java | 6 +- .../gen/CockroachDBSetSessionGenerator.java | 3 +- .../gen/CockroachDBTableGenerator.java | 30 +++------- .../gen/CockroachDBTruncateGenerator.java | 2 +- .../oracle/tlp/CockroachDBTLPBase.java | 2 +- .../tlp/CockroachDBTLPHavingOracle.java | 5 +- .../common/query/SQLQueryAdapter.java | 12 +--- src/sqlancer/tidb/TiDBBugs.java | 40 ++++++++----- src/sqlancer/tidb/TiDBErrors.java | 42 +++++++++---- .../tidb/TiDBExpressionGenerator.java | 43 +++++++++---- src/sqlancer/tidb/TiDBSchema.java | 34 +++++++++++ .../ast/TiDBBinaryComparisonOperation.java | 4 +- src/sqlancer/tidb/ast/TiDBFunctionCall.java | 5 +- src/sqlancer/tidb/ast/TiDBRegexOperation.java | 5 +- .../tidb/gen/TiDBAlterTableGenerator.java | 11 ++-- .../tidb/gen/TiDBAnalyzeTableGenerator.java | 9 +-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 5 +- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 53 +++++----------- .../tidb/gen/TiDBUpdateGenerator.java | 2 +- 29 files changed, 316 insertions(+), 186 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 145342ac4..016f41f75 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -57,16 +57,17 @@ public static List getResultSetFirstColumnAsString(String queryString, E throw new IgnoreMeException(); } while (result.next()) { - resultSet.add(result.getString(1)); + String resultTemp = result.getString(1); + if (resultTemp != null) { + resultTemp = resultTemp.replaceAll("[\\.]0+$", ""); + } + resultSet.add(resultTemp); // Remove the trailing zeros as many DBMS treat it as non-bugs } } catch (Exception e) { if (e instanceof IgnoreMeException) { throw e; } - if (e instanceof NumberFormatException) { - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/57 - throw new IgnoreMeException(); - } + if (e.getMessage() == null) { throw new AssertionError(queryString, e); } diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index c26e8aa17..fa317f1f0 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -159,13 +159,17 @@ public static List extractNrRandomColumns(List columns, int nr) { public static int smallNumber() { // no need to cache for small numbers - return (int) (Math.abs(getThreadRandom().get().nextGaussian()) * 2); + return (int) (Math.abs(getThreadRandom().get().nextGaussian())) * 2; } public static boolean getBoolean() { return getThreadRandom().get().nextBoolean(); } + public static double getPercentage() { + return getThreadRandom().get().nextDouble(); + } + private static ThreadLocal getThreadRandom() { if (THREAD_RANDOM.get() == null) { // a static method has been called, before Randomly was instantiated diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java index 1469ec0da..4f5a089ca 100644 --- a/src/sqlancer/SQLProviderAdapter.java +++ b/src/sqlancer/SQLProviderAdapter.java @@ -25,9 +25,15 @@ protected void checkViewsAreValid(G globalState) { for (AbstractTable view : views) { SQLQueryAdapter q = new SQLQueryAdapter("SELECT 1 FROM " + view.getName() + " LIMIT 1"); try { - q.execute(globalState); + if (!q.execute(globalState)) { + throw new AssertionError(); + } } catch (Throwable t) { - throw new IgnoreMeException(); + try { + globalState.executeStatement(new SQLQueryAdapter("DROP VIEW " + view.getName(), true)); + } catch (Throwable t2) { + throw new IgnoreMeException(); + } } } } diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 332f51c4c..3a980a8c7 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -3,13 +3,52 @@ public final class CockroachDBBugs { // https://github.com/cockroachdb/cockroach/issues/46915 - public static boolean bug46915 = true; + public static boolean bug46915; // https://github.com/cockroachdb/cockroach/issues/45703 - public static boolean bug45703 = true; + public static boolean bug45703; // https://github.com/cockroachdb/cockroach/issues/44757 - public static boolean bug44757 = true; + public static boolean bug44757; + + // https://github.com/cockroachdb/cockroach/issues/83792 + public static boolean bug83792 = true; + + // https://github.com/cockroachdb/cockroach/issues/83874 + public static boolean bug83874 = true; + + // https://github.com/cockroachdb/cockroach/issues/83973 + public static boolean bug83973 = true; + + // https://github.com/cockroachdb/cockroach/issues/83976 + public static boolean bug83976 = true; + + // https://github.com/cockroachdb/cockroach/issues/84154 + public static boolean bug84154 = true; + + // https://github.com/cockroachdb/cockroach/issues/85356 + public static boolean bug85356 = true; + + // https://github.com/cockroachdb/cockroach/issues/85371 + public static boolean bug85371 = true; + + // https://github.com/cockroachdb/cockroach/issues/85389 + public static boolean bug85389 = true; + + // https://github.com/cockroachdb/cockroach/issues/85390 + public static boolean bug85390 = true; + + // https://github.com/cockroachdb/cockroach/issues/85393 + public static boolean bug85393 = true; + + // https://github.com/cockroachdb/cockroach/issues/85394 + public static boolean bug85394 = true; + + // https://github.com/cockroachdb/cockroach/issues/85441 + public static boolean bug85441 = true; + + // https://github.com/cockroachdb/cockroach/issues/85499 + public static boolean bug85499 = true; private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 14c74d039..8fd79cb99 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -33,9 +33,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("expected -9223372036854775809 to be of type int, found type decimal"); errors.add("to be of type int4, found type decimal"); - errors.add("as type bool: invalid bool value"); - errors.add("as type int: strconv.ParseInt"); - errors.add("as type float: strconv.ParseFloat: parsing"); + errors.add("as type bool"); + errors.add("as type int"); + errors.add("as type float"); errors.add("is not in select list"); errors.add("non-integer constant in ORDER BY"); @@ -68,9 +68,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("unsupported comparison operator: = "); errors.add("unsupported comparison operator: <= "); errors.add("to be of type string, found type bytes"); - errors.add("unknown signature: left(string, int) (desired )"); errors.add("unknown signature: bit_length(collatedstring"); - errors.add("ERROR: unknown signature: left(collatedstring"); errors.add("unsupported comparison operator: !~ NOT LIKE )"); + errors.add("unknown signature: left"); errors.add("numeric constant out of int64 range"); - errors.add("unknown signature: overlay(string, string, decimal)"); - errors.add("unknown signature: substring(string, int, decimal)"); + errors.add("unknown signature: overlay"); + errors.add("unknown signature: substring"); errors.add("unsupported binary operator: + (desired )"); errors.add("unsupported comparison operator"); errors.add("unknown signature: chr(decimal) (desired )"); @@ -122,16 +119,13 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("incompatible value type: expected rowid to be of type decimal, found type int"); errors.add("unknown signature: to_english(decimal)"); errors.add("unknown signature: chr(decimal)"); - errors.add(" unknown signature: left(string, int2) (desired )"); - errors.add("unknown signature: split_part(string, string, decimal) (desired )"); - errors.add(" unknown signature: substring(string, "); + errors.add("unknown signature: split_part"); errors.add("division by zero"); errors.add("as int, found type: decimal"); errors.add("value type decimal doesn't match type int2 "); errors.add("has type decimal"); errors.add("to be of type decimal, found type int"); errors.add("value type decimal doesn't match type int"); - errors.add("unknown signature: substring(string, decimal, int)"); errors.add("unsupported binary operator: / (desired )"); errors.add("(desired )"); errors.add("(desired )"); @@ -139,8 +133,6 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("found type: decimal"); errors.add("(desired )"); errors.add("unknown signature: to_hex(decimal)"); - errors.add("unknown signature: split_part(string, string, decimal)"); - errors.add("unknown signature: left(bytes, decimal)"); errors.add("division undefined"); errors.add("decimal out of range"); errors.add("unknown signature: xor_agg(decimal)"); @@ -155,14 +147,45 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("could not parse JSON: unable to decode JSON: EOF"); errors.add("could not parse JSON: unable to decode JSON: unexpected EOF"); errors.add("can't order by column type jsonb"); + errors.add("odd length hex string"); // TODO: better control what is generated in a view errors.add("aggregate functions are not allowed in GROUP BY"); errors.add(" must appear in the GROUP BY clause or be used in an aggregate function"); - if (CockroachDBBugs.bug44757) { + if (CockroachDBBugs.bug83874) { errors.add("no builtin aggregate"); } + if (CockroachDBBugs.bug83792) { + errors.add("comparison overload not found"); + } + if (CockroachDBBugs.bug83973) { + errors.add("invalid memory address"); + } + if (CockroachDBBugs.bug83976) { + errors.add("cannot execute distinct on no columns"); + } + if (CockroachDBBugs.bug85356) { + errors.add("inconsistent Case return types"); + } + if (CockroachDBBugs.bug85371) { + errors.add("index out of range"); + } + if (CockroachDBBugs.bug85389) { + errors.add("no volatility for cast decimal"); + } + if (CockroachDBBugs.bug85390) { + errors.add("lookup for ComparisonExpr"); + } + if (CockroachDBBugs.bug85393) { + errors.add("no output column equivalent to"); + } + if (CockroachDBBugs.bug85441) { + errors.add("cannot cast jsonb numeric to type bool"); + } + if (CockroachDBBugs.bug85499) { + errors.add("estimated row count must be non-zero"); + } errors.add("unable to vectorize execution plan"); // SET vectorize=experimental_always; errors.add(" mismatched physical types at index"); // SET vectorize=experimental_always; @@ -223,7 +246,7 @@ private static void addArrayErrors(ExpectedErrors errors) { errors.add("unimplemented: nested arrays not supported"); // e.g., casting a string {{1}} to an array errors.add("malformed array"); - errors.add("https://github.com/cockroachdb/cockroach/issues/35707"); // arrays don't support ORDER BY + // errors.add("https://github.com/cockroachdb/cockroach/issues/35707"); // arrays don't support ORDER BY errors.add("as bytes[], found type: varbit[]"); errors.add("to be of type decimal[], found type float[]"); @@ -234,7 +257,7 @@ private static void addArrayErrors(ExpectedErrors errors) { private static void addIntervalTypeErrors(ExpectedErrors errors) { errors.add("overflow during Encode"); - errors.add("as type interval"); + errors.add("type interval"); } private static void addJoinTypes(ExpectedErrors errors) { @@ -269,7 +292,6 @@ private static void addFunctionErrors(ExpectedErrors errors) { // functions errors.add("abs of min integer value (-9223372036854775808) not defined"); // ABS errors.add("the input string must not be empty"); // ASCII - errors.add("unknown signature: substring(string, decimal)"); // overflow errors.add("overlay(): non-positive substring length not allowed"); // overlay errors.add("non-positive substring length not allowed"); // overlay errors.add("lpad(): requested length too large"); // lpad diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index f8ab8e07f..aa41259b0 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -97,7 +97,7 @@ public TestOracle create(CockroachDBGlobalState globalState) throws SQLException public boolean testHashIndexes = true; @Parameter(names = { "--test-temp-tables" }, description = "Test TEMPORARY tables") - public boolean testTempTables = true; + public boolean testTempTables; // default: false https://github.com/cockroachdb/cockroach/issues/85388 @Parameter(names = { "--increased-vectorization" }, description = "Generate VECTORIZE=on with a higher probability (which found a number of bugs in the past)") diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 5c21a1107..1c1dd3531 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -274,6 +274,14 @@ public CockroachDBTable(String tableName, List columns, List< } + public int getIndexCount() { + int count = 0; + for (CockroachDBTable table : getDatabaseTables()) { + count += table.getIndexes().size(); + } + return count; + } + public static CockroachDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); @@ -285,6 +293,10 @@ public static CockroachDBSchema fromConnection(SQLConnection con, String databas for (CockroachDBColumn c : databaseColumns) { c.setTable(t); } + // To avoid some situations that columns can not be got. + if (databaseColumns.isEmpty()) { + continue; + } databaseTables.add(t); } @@ -334,6 +346,11 @@ private static List getTableColumns(SQLConnection con, String isNullable); columns.add(c); } + } catch (SQLException e) { + if (CockroachDBBugs.bug85394 && e.getMessage().contains("incompatible type annotation for ARRAY")) { + return columns; + } + throw e; } } return columns; diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java b/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java index 84e2cb996..aa4dabcc4 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java @@ -230,20 +230,22 @@ public static CockroachDBExpression createBitConstantWithSize(int size) { } public static CockroachDBExpression createTimestampConstant(long integer) { - return new CockroachDBTimeRelatedConstant("TIMESTAMP", integer, "yyyy-MM-dd"); + return new CockroachDBTimeRelatedConstant("TIMESTAMP", integer, "yyyy-MM-dd'T'HH:mm:ss"); } public static CockroachDBExpression createTimeConstant(long integer) { - return new CockroachDBTimeRelatedConstant("TIME", integer, "HH:mm:ss"); + return new CockroachDBTimeRelatedConstant("TIME", integer, "yyyy-MM-dd'T'HH:mm:ss"); } public static CockroachDBExpression createTimetz(long integer) { - return new CockroachDBTimeRelatedConstant("TIMETZ", integer, "HH:mm:ss"); // TODO: support the complete format + return new CockroachDBTimeRelatedConstant("TIMETZ", integer, "yyyy-MM-dd'T'HH:mm:ss"); // TODO: support the + // complete format } public static CockroachDBExpression createTimestamptzConstant(long integer) { - return new CockroachDBTimeRelatedConstant("TIMESTAMPTZ", integer, "HH:mm:ss"); // TODO: support the complete - // format + return new CockroachDBTimeRelatedConstant("TIMESTAMPTZ", integer, "yyyy-MM-dd'T'HH:mm:ss"); // TODO: support the + // complete + // format } public static CockroachDBExpression createIntervalConstant(long year, long month, long day, long hour, long minute, diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java index 81920cd4f..b8895a34e 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBCreateStatisticsGenerator.java @@ -22,12 +22,7 @@ public static SQLQueryAdapter create(CockroachDBGlobalState globalState) { sb.append(" FROM "); sb.append(randomTable.getName()); - return new SQLQueryAdapter(sb.toString(), - ExpectedErrors.from("current transaction is aborted, commands ignored until end of transaction block", - "ERROR: unable to encode table key: *tree.DArray" /* - * https://github.com/cockroachdb/cockroach/ - * issues/46964 - */, "overflow during Encode")); + return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("overflow during Encode")); // https://github.com/cockroachdb/cockroach/issues/84078 } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 66c2936fd..e5b708338 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -3,6 +3,7 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; @@ -30,6 +31,10 @@ public void buildStatement() { errors.add("the following columns are not indexable due to their type"); // array types are not indexable errors.add("cannot determine type of empty array. Consider annotating with the desired type"); errors.add("incompatible IF expression"); // TODO: investigate; seems to be a bug + if (CockroachDBBugs.bug84154) { + errors.add("overflow during Encode"); + errors.add("of type interval"); + } CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append("CREATE "); if (Randomly.getBoolean()) { @@ -43,15 +48,19 @@ public void buildStatement() { && Randomly.getBooleanWithSmallProbability(); if (hashSharded) { sb.append(" USING HASH WITH BUCKET_COUNT="); - sb.append(Randomly.getNotCachedInteger(2, Short.MAX_VALUE)); + sb.append(Randomly.getNotCachedInteger(2, 2048)); errors.add("null value in column"); errors.add("cannot create a sharded index on a computed column"); } if (Randomly.getBoolean()) { - sb.append(" "); - sb.append(Randomly.fromOptions("STORING", "COVERING")); - sb.append(" "); - addColumns(sb, table.getRandomNonEmptyColumnSubset(), false); + List columns2 = table.getRandomNonEmptyColumnSubset(); + columns2.removeAll(columns); + if (!columns2.isEmpty()) { + sb.append(" "); + sb.append(Randomly.fromOptions("STORING", "COVERING")); + sb.append(" "); + addColumns(sb, columns2, false); + } } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 2d411f26e..cef9862de 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -15,12 +15,12 @@ private CockroachDBSetClusterSettingGenerator() { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBClusterSetting { - BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), + // BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), - SQL_DEFAULTS_VECTORIZE_ROW_COUNT_THRESHOLD("sql.defaults.vectorize_row_count_threshold", - (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), + // SQL_DEFAULTS_VECTORIZE_ROW_COUNT_THRESHOLD("sql.defaults.vectorize_row_count_threshold", + // (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), // SQL_DEFAULTS_EXPERIMENTAL_OPTIMIZER_FOREIGN_KEYS_ENABLED("sql.defaults.experimental_optimizer_foreign_keys.enabled", // CockroachDBSetSessionGenerator::onOff), SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index 48bb42fb7..e99eaaa27 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -29,8 +29,7 @@ private enum CockroachDBSetting { SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), // EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), REORDER_JOINS_LIMIT((g) -> g.getRandomly().getInteger(0, Integer.MAX_VALUE)), // - SQL_SAFE_UPDATES((g) -> "off"), - // TRACING(CockroachDBSetSessionGenerator::onOff) + SQL_SAFE_UPDATES((g) -> "off"), TRACING(CockroachDBSetSessionGenerator::onOff), /* * CockroachDB enables vectorized (column-oriented) execution by default. Row-oriented execution can be enforced * by setting vectorized to "off". Some examples of bugs found in the vectorized execution engine are: diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index 720f2a300..4e99ab31a 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -72,7 +72,13 @@ public void buildStatement() { && cockroachDBColumn.getType().getPrimitiveDataType() != CockroachDBDataType.SERIAL; if (generatedColumn) { sb.append(" AS ("); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(cockroachDBColumn.getType()))); + // To generate an expression exclude of the current column + List generatedColumns = new ArrayList<>(columns); + generatedColumns.remove(i); + CockroachDBExpressionGenerator genGeneratedColumn = new CockroachDBExpressionGenerator(globalState) + .setColumns(generatedColumns); + sb.append(CockroachDBVisitor + .asString(genGeneratedColumn.generateExpression(cockroachDBColumn.getType()))); sb.append(") STORED"); errors.add("computed columns cannot reference other computed columns"); errors.add("context-dependent operators are not allowed in computed column"); @@ -144,28 +150,6 @@ public void buildStatement() { sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); sb.append(")"); } - if (Randomly.getBoolean() && !globalState.getSchema().getDatabaseTables().isEmpty()) { - sb.append(", "); - // TODO: also allow referencing itself - List subset = Randomly.nonEmptySubset(columns); - sb.append(" FOREIGN KEY ("); - sb.append(subset.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(") REFERENCES "); - CockroachDBTable otherTable = globalState.getSchema().getRandomTable(); - sb.append(otherTable.getName()); - sb.append("("); - for (int i = 0; i < subset.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(otherTable.getRandomColumn().getName()); - } - sb.append(")"); - // TODO: ensure that the column types match - errors.add("does not match foreign key"); - errors.add("computed column"); - errors.add("there is no unique constraint matching given keys for referenced table"); - } sb.append(")"); errors.add("collatedstring"); CockroachDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index 02579cf32..d10000352 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -16,7 +16,7 @@ public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { errors.add("is referenced by foreign key"); // https://github.com/cockroachdb/cockroach/issues/47030 - errors.add("unexpected value: "); + // errors.add("unexpected value: "); StringBuilder sb = new StringBuilder(); sb.append("TRUNCATE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index e08389bf5..7e7f03e26 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -54,7 +54,7 @@ public void check() throws SQLException { List generateFetchColumns() { List columns = new ArrayList<>(); - if (Randomly.getBoolean()) { + if (Randomly.getBoolean() || targetTables.getColumns().size() == 0) { columns.add(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); } else { columns.addAll(Randomly.nonEmptySubset(targetTables.getColumns()).stream() diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java index d5fcf9812..42abad00b 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java @@ -33,11 +33,12 @@ public void check() throws SQLException { String originalQueryString = CockroachDBVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); select.setHavingClause(predicate); String firstQueryString = CockroachDBVisitor.asString(select); - select.setHavingClause(negatedPredicate); + select.setHavingClause(gen.negatePredicate(predicate)); String secondQueryString = CockroachDBVisitor.asString(select); - select.setHavingClause(isNullPredicate); + select.setHavingClause(gen.isNull(predicate)); String thirdQueryString = CockroachDBVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 01fcc8539..c2e6706cc 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -96,17 +96,9 @@ public String getUnterminatedQueryString() { } public void checkException(Exception e) throws AssertionError { - Throwable ex = e; - - while (ex != null) { - if (expectedErrors.errorIsExpected(ex.getMessage())) { - return; - } else { - ex = ex.getCause(); - } + if (!expectedErrors.errorIsExpected(e.getMessage())) { + throw new AssertionError(query, e); } - - throw new AssertionError(query, e); } @Override diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index 08d5d423d..a4f5cc2dd 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -4,46 +4,58 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/15987 - public static boolean bug15987 = true; + public static boolean bug15987; // // https://github.com/pingcap/tidb/issues/15988 - public static boolean bug15988 = true; + public static boolean bug15988; // https://github.com/pingcap/tidb/issues/16028 - public static boolean bug16028 = true; + public static boolean bug16028; // https://github.com/pingcap/tidb/issues/16020 - public static boolean bug16020 = true; + public static boolean bug16020; // https://github.com/pingcap/tidb/issues/15990 - public static boolean bug15990 = true; + public static boolean bug15990; // https://github.com/pingcap/tidb/issues/15844 - public static boolean bug15844 = true; + public static boolean bug15844; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/10 - public static boolean bug10 = true; + public static boolean bug10; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/14 - public static boolean bug14 = true; + public static boolean bug14; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/15 - public static boolean bug15 = true; + public static boolean bug15; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/16 - public static boolean bug16 = true; + public static boolean bug16; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/19 - public static boolean bug19 = true; + public static boolean bug19; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/48 - public static boolean bug48 = true; + public static boolean bug48; // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/50 - public static boolean bug50 = true; + public static boolean bug50; // https://github.com/citusdata/citus/issues/4079 - public static boolean bug4079 = true; + public static boolean bug4079; + + // https://github.com/pingcap/tidb/issues/35677 + public static boolean bug35677 = true; + + // https://github.com/pingcap/tidb/issues/35522 + public static boolean bug35522 = true; + + // https://github.com/pingcap/tidb/issues/35652 + public static boolean bug35652 = true; + + // https://github.com/pingcap/tidb/issues/38295 + public static boolean bug38295 = true; private TiDBBugs() { } diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index bb3d3ca62..95979264c 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -25,13 +25,14 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("is not valid for CHARACTER SET"); // known issue: https://github.com/pingcap/tidb/issues/14819 - errors.add("Wrong plan type for dataReaderBuilder"); + // errors.add("Wrong plan type for dataReaderBuilder"); errors.add("DOUBLE value is out of range"); - errors.add("index out of range"); // https://github.com/pingcap/tidb/issues/15810 - errors.add("baseBuiltinFunc.evalString() should never be called, please contact the TiDB team for help"); // https://github.com/pingcap/tidb/issues/15847 - errors.add("unsupport column type for encode 6"); // https://github.com/pingcap/tidb/issues/15850 + // errors.add("index out of range"); // https://github.com/pingcap/tidb/issues/15810 + // errors.add("baseBuiltinFunc.evalString() should never be called, please contact the TiDB team for help"); // + // https://github.com/pingcap/tidb/issues/15847 + // errors.add("unsupport column type for encode 6"); // https://github.com/pingcap/tidb/issues/15850 errors.add("Data truncation: %s value is out of range in '%s'"); errors.add("Truncated incorrect FLOAT value"); @@ -39,13 +40,32 @@ public static void addExpressionErrors(ExpectedErrors errors) { // regex errors.add("error parsing regexp"); + errors.add("from regexp"); + + // To avoid bugs + errors.add("Unknown column"); // https://github.com/pingcap/tidb/issues/35522 + errors.add("Can\'t find column"); // https://github.com/pingcap/tidb/issues/35527 + errors.add("Cannot convert"); // https://github.com/pingcap/tidb/issues/35652 // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/57 - errors.add("For input string: \"+Inf\""); + // errors.add("For input string: \"+Inf\""); + + // errors.add("inconsistent index"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/58 - errors.add("inconsistent index"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/58 + // errors.add("Illegal mix of collations"); - errors.add("Illegal mix of collations"); + if (TiDBBugs.bug35677) { + errors.add("for function inet_aton"); + } + if (TiDBBugs.bug35522) { + errors.add("ERROR 1054 (42S22)"); + } + if (TiDBBugs.bug35652) { + errors.add("from binary to utf8"); + } + if (TiDBBugs.bug38295) { + errors.add("assertion failed"); + } } public static void addExpressionHavingErrors(ExpectedErrors errors) { @@ -73,10 +93,10 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("error parsing regexp"); errors.add("is not valid for CHARACTER SET"); - if (true) { - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/54 - errors.add("Miss column"); - } + // if (true) { + // // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/54 + // errors.add("Miss column"); + // } } } diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index a960363fe..20f48f04a 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -20,7 +20,6 @@ import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; import sqlancer.tidb.ast.TiDBCase; import sqlancer.tidb.ast.TiDBCastOperation; -import sqlancer.tidb.ast.TiDBCollate; import sqlancer.tidb.ast.TiDBColumnReference; import sqlancer.tidb.ast.TiDBConstant; import sqlancer.tidb.ast.TiDBExpression; @@ -47,7 +46,7 @@ private enum Gen { UNARY_POSTFIX, // CONSTANT, // COLUMN, // - COMPARISON, REGEX, COLLATE, FUNCTION, BINARY_LOGICAL, BINARY_BIT, CAST, DEFAULT, CASE + COMPARISON, REGEX, FUNCTION, BINARY_LOGICAL, BINARY_BIT, CAST, DEFAULT, CASE // BINARY_ARITHMETIC } @@ -86,9 +85,9 @@ protected TiDBExpression generateExpression(int depth) { case REGEX: return new TiDBRegexOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBRegexOperator.getRandom()); - case COLLATE: - return new TiDBCollate(generateExpression(depth + 1), - Randomly.fromOptions("utf8mb4_bin", "latin1_bin", "binary", "ascii_bin", "utf8_bin")); + // case COLLATE: + // return new TiDBCollate(generateExpression(depth + 1), + // Randomly.fromOptions("utf8mb4_bin", "latin1_bin", "binary", "ascii_bin", "utf8_bin")); case FUNCTION: TiDBFunction func = TiDBFunction.getRandom(); return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); @@ -105,13 +104,9 @@ protected TiDBExpression generateExpression(int depth) { // return new TiDBBinaryArithmeticOperation(generateExpression(depth + 1), generateExpression(depth + 1), // TiDBBinaryArithmeticOperator.getRandom()); case CAST: - return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions( - /* - * "BINARY" https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 - */ "CHAR", - /* - * "DATE", "DATETIME", "TIME", https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 - */ "DECIMAL", "SIGNED"/* , "UNSIGNED" https://github.com/pingcap/tidb/issues/16028 */)); + return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions("BINARY", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 + "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 + "DECIMAL", "SIGNED", "UNSIGNED" /* https://github.com/pingcap/tidb/issues/16028 */)); case CASE: if (TiDBBugs.bug19) { throw new IgnoreMeException(); @@ -180,4 +175,28 @@ public TiDBExpression isNull(TiDBExpression expr) { return new TiDBUnaryPostfixOperation(expr, TiDBUnaryPostfixOperator.IS_NULL); } + public TiDBExpression generateConstant(TiDBDataType type) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return TiDBConstant.createNullConstant(); + } + switch (type) { + case INT: + return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + case BLOB: + case TEXT: + return TiDBConstant.createStringConstant(globalState.getRandomly().getString()); + case BOOL: + return TiDBConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOATING: + return TiDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); + case CHAR: + return TiDBConstant.createStringConstant(globalState.getRandomly().getChar()); + case DECIMAL: + case NUMERIC: + return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(); + } + } + } diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index a611543fc..b147ab8ab 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -56,6 +56,23 @@ public boolean isNumeric() { throw new AssertionError(this); } } + + public boolean canHaveDefault() { + switch (this) { + case INT: + case DECIMAL: + case FLOATING: + case BOOL: + case CHAR: + return true; + case NUMERIC: + case TEXT: + case BLOB: + return false; + default: + throw new AssertionError(this); + } + } } public static class TiDBCompositeDataType { @@ -176,6 +193,14 @@ public TiDBTables getRandomTableNonEmptyTables() { return new TiDBTables(Randomly.nonEmptySubset(getDatabaseTables())); } + public int getIndexCount() { + int count = 0; + for (TiDBTable table : getDatabaseTables()) { + count += table.getIndexes().size(); + } + return count; + } + private static TiDBCompositeDataType getColumnType(String typeString) { String trimmedStringType = typeString.replace(" zerofill", "").replace(" unsigned", ""); if (trimmedStringType.contains("decimal")) { @@ -211,14 +236,17 @@ private static TiDBCompositeDataType getColumnType(String typeString) { case "null": primitiveType = TiDBDataType.INT; break; + case "tinyint(3)": case "tinyint(4)": primitiveType = TiDBDataType.INT; size = 1; break; + case "smallint(5)": case "smallint(6)": primitiveType = TiDBDataType.INT; size = 2; break; + case "int(10)": case "int(11)": primitiveType = TiDBDataType.INT; size = 4; @@ -252,6 +280,10 @@ public static TiDBSchema fromConnection(SQLConnection con, String databaseName) List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); + // Ignore invalid views + if (databaseColumns.isEmpty()) { + continue; + } List indexes = getIndexes(con, tableName); boolean isView = tableName.startsWith("v"); TiDBTable t = new TiDBTable(tableName, databaseColumns, indexes, isView); @@ -302,6 +334,8 @@ private static List getTableColumns(SQLConnection con, String tableN columns.add(c); } } + } catch (SQLException e) { + } return columns; } diff --git a/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java b/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java index 4efc554f0..c7240e4b7 100644 --- a/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java +++ b/src/sqlancer/tidb/ast/TiDBBinaryComparisonOperation.java @@ -14,8 +14,8 @@ public enum TiDBComparisonOperator implements Operator { GREATER_EQUALS(">="), // SMALLER("<"), // SMALLER_EQUALS("<="), // - NOT_EQUALS("!="); // - // NULL_SAFE_EQUALS("<=>"); https://github.com/tidb-challenge-program/bug-hunting-issue/issues/5 + NOT_EQUALS("!="), // + NULL_SAFE_EQUALS("<=>"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/5 private String textRepr; diff --git a/src/sqlancer/tidb/ast/TiDBFunctionCall.java b/src/sqlancer/tidb/ast/TiDBFunctionCall.java index 4aded9ad2..1aaa87827 100644 --- a/src/sqlancer/tidb/ast/TiDBFunctionCall.java +++ b/src/sqlancer/tidb/ast/TiDBFunctionCall.java @@ -92,11 +92,10 @@ public int getNrArgs() { // [...] REPLACE(3), // REVERSE(1), // - RIGHT(2), // + // RIGHT(2), // https://github.com/pingcap/tidb/issues/35623 // RPAD TODO RTRIM(1), // - // SPACE(1), - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/6 + SPACE(1), // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/6 STRCMP(2), // SUBSTRING(2), // TODO: support other versions SUBSTRING_INDEX(3), // diff --git a/src/sqlancer/tidb/ast/TiDBRegexOperation.java b/src/sqlancer/tidb/ast/TiDBRegexOperation.java index f5f011dd6..c8da018d8 100644 --- a/src/sqlancer/tidb/ast/TiDBRegexOperation.java +++ b/src/sqlancer/tidb/ast/TiDBRegexOperation.java @@ -9,8 +9,9 @@ public class TiDBRegexOperation extends BinaryOperatorNode !t.isView()); TiDBColumn column = table.getRandomColumn(); @@ -58,9 +64,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append(Randomly.fromOptions("ENABLE", "DISABLE")); sb.append(" KEYS"); break; - case FORCE: - sb.append("FORCE"); - break; case DROP_PRIMARY_KEY: if (!column.isPrimaryKey()) { throw new IgnoreMeException(); diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index b52e2cc25..d84b117c2 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -3,7 +3,6 @@ import java.sql.SQLException; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -31,13 +30,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(Randomly.getNotCachedInteger(1, 1024)); sb.append(" BUCKETS"); } - return new SQLQueryAdapter(sb.toString(), - ExpectedErrors.from("https://github.com/pingcap/tidb/issues/15993", - /* https://github.com/pingcap/tidb/issues/15993 */ "doesn't have a default value", - "Index 'PRIMARY' in field list does not exist in table" /* - * https://github. com/pingcap/tidb/ - * issues/15993 - */)); + return new SQLQueryAdapter(sb.toString()); } } diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 5ed96f707..b48ea31cc 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -45,9 +45,8 @@ private enum Action { TIDB_ENABLE_FAST_ANALYZE("tidb_enable_fast_analyze", (r) -> Randomly.fromOptions(0, 1)), // TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), - // TODO: global - // TIDB_SCATTER_REGION("tidb_scatter_region", (r) -> Randomly.fromOptions(0, 1)); - TIDB_ENABLE_STMT_SUMMARY("tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // + TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions(0, 1)), + TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // TIDB_ENABLE_CHUNK_RPC("tidb_enable_chunk_rpc", (r) -> Randomly.fromOptions(0, 1)); private String name; diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index c252a7b9c..97859d1ad 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -59,10 +58,7 @@ private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { sb.append(columns.get(i).getName()); sb.append(" "); TiDBCompositeDataType type; - do { - type = TiDBCompositeDataType.getRandom(); - } while (type.getPrimitiveDataType() == TiDBDataType.INT && type.getSize() < 4 - || type.getPrimitiveDataType() == TiDBDataType.BOOL); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/49 + type = TiDBCompositeDataType.getRandom(); appendType(sb, type); sb.append(" "); boolean isGeneratedColumn = Randomly.getBooleanWithRatherLowProbability(); @@ -72,7 +68,6 @@ private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { sb.append(") "); sb.append(Randomly.fromOptions("STORED", "VIRTUAL")); sb.append(" "); - errors.add("You have an error in your SQL syntax"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/53 errors.add("Generated column can refer only to generated columns defined prior to it"); errors.add( "'Defining a virtual generated column as primary key' is not supported for generated columns."); @@ -87,10 +82,9 @@ private void createNewTable(TiDBExpressionGenerator gen, StringBuilder sb) { if (Randomly.getBooleanWithRatherLowProbability()) { sb.append("NOT NULL "); } - if (Randomly.getBoolean() && type.getPrimitiveDataType() != TiDBDataType.TEXT - && type.getPrimitiveDataType() != TiDBDataType.BLOB && !isGeneratedColumn) { + if (Randomly.getBoolean() && type.getPrimitiveDataType().canHaveDefault() && !isGeneratedColumn) { sb.append("DEFAULT "); - sb.append(TiDBVisitor.asString(gen.generateConstant())); + sb.append(TiDBVisitor.asString(gen.generateConstant(type.getPrimitiveDataType()))); sb.append(" "); errors.add("Invalid default value"); errors.add( @@ -137,27 +131,16 @@ && canUseAsUnique(type) && !isGeneratedColumn) { errors.add("UnknownType: *ast.WhenClause"); } } - List actions = Randomly.nonEmptySubset(Action.values()); - for (Action a : actions) { - sb.append(" "); - switch (a) { - case AUTO_INCREMENT: - sb.append("AUTO_INCREMENT="); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - break; - case PRE_SPLIT_REGIONS: - sb.append("PRE_SPLIT_REGIONS="); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - break; - case SHARD_ROW_ID_BITS: - sb.append("SHARD_ROW_ID_BITS="); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - errors.add("Unsupported shard_row_id_bits for table with primary key as row id"); - break; - default: - throw new AssertionError(a); - } - } + // Buggy code and disable it temporarily + /* + * List actions = Randomly.nonEmptySubset(Action.values()); for (Action a : actions) { sb.append(" "); + * switch (a) { case AUTO_INCREMENT: sb.append("AUTO_INCREMENT="); sb.append(Randomly.getNotCachedInteger(0, + * Integer.MAX_VALUE)); break; case PRE_SPLIT_REGIONS: sb.append("PRE_SPLIT_REGIONS="); + * sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); break; case SHARD_ROW_ID_BITS: + * sb.append("SHARD_ROW_ID_BITS="); sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); + * errors.add("Unsupported shard_row_id_bits for table with primary key as row id"); break; default: throw new + * AssertionError(a); } } + */ } private boolean canUseAsUnique(TiDBCompositeDataType type) { @@ -165,18 +148,14 @@ private boolean canUseAsUnique(TiDBCompositeDataType type) { } private void appendType(StringBuilder sb, TiDBCompositeDataType type) { - if (type.getPrimitiveDataType() == TiDBDataType.CHAR) { - throw new IgnoreMeException(); - } + // if (type.getPrimitiveDataType() == TiDBDataType.CHAR) { + // throw new IgnoreMeException(); + // } sb.append(type.toString()); appendSpecifiers(sb, type.getPrimitiveDataType()); appendSizeSpecifiers(sb, type.getPrimitiveDataType()); } - private enum Action { - AUTO_INCREMENT, PRE_SPLIT_REGIONS, SHARD_ROW_ID_BITS - } - private void appendSizeSpecifiers(StringBuilder sb, TiDBDataType type) { if (type.isNumeric() && Randomly.getBoolean() && !TiDBBugs.bug16028) { sb.append(" UNSIGNED"); diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 38eba8e78..0c2459b28 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -43,7 +43,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); sb.append(TiDBVisitor.asString(gen.generateExpression())); - errors.add("Data Too Long"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/43 + // errors.add("Data Too Long"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/43 } TiDBErrors.addInsertErrors(errors); From 1bbbe7989fd10b2a89cd329df3cc0e49a0b7094a Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Wed, 5 Oct 2022 10:52:04 +0800 Subject: [PATCH 189/856] clean comments --- src/sqlancer/ComparatorHelper.java | 5 +++-- src/sqlancer/SQLProviderAdapter.java | 16 +++++++++----- .../cockroachdb/CockroachDBErrors.java | 2 -- .../cockroachdb/CockroachDBSchema.java | 2 +- ...CockroachDBSetClusterSettingGenerator.java | 5 ----- .../gen/CockroachDBSetSessionGenerator.java | 2 -- .../gen/CockroachDBTruncateGenerator.java | 2 -- .../common/query/SQLQueryAdapter.java | 12 ++++++++-- src/sqlancer/tidb/TiDBErrors.java | 22 ------------------- src/sqlancer/tidb/TiDBSchema.java | 3 +-- src/sqlancer/tidb/ast/TiDBFunctionCall.java | 2 +- src/sqlancer/tidb/ast/TiDBRegexOperation.java | 5 ++--- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 13 ----------- .../tidb/gen/TiDBUpdateGenerator.java | 1 - 14 files changed, 28 insertions(+), 64 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 016f41f75..880d3ae72 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -59,9 +59,10 @@ public static List getResultSetFirstColumnAsString(String queryString, E while (result.next()) { String resultTemp = result.getString(1); if (resultTemp != null) { - resultTemp = resultTemp.replaceAll("[\\.]0+$", ""); + resultTemp = resultTemp.replaceAll("[\\.]0+$", ""); // Remove the trailing zeros as many DBMS treat + // it as non-bugs } - resultSet.add(resultTemp); // Remove the trailing zeros as many DBMS treat it as non-bugs + resultSet.add(resultTemp); } } catch (Exception e) { if (e instanceof IgnoreMeException) { diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java index 4f5a089ca..8de7523a3 100644 --- a/src/sqlancer/SQLProviderAdapter.java +++ b/src/sqlancer/SQLProviderAdapter.java @@ -26,15 +26,19 @@ protected void checkViewsAreValid(G globalState) { SQLQueryAdapter q = new SQLQueryAdapter("SELECT 1 FROM " + view.getName() + " LIMIT 1"); try { if (!q.execute(globalState)) { - throw new AssertionError(); + dropView(globalState, view.getName()); } } catch (Throwable t) { - try { - globalState.executeStatement(new SQLQueryAdapter("DROP VIEW " + view.getName(), true)); - } catch (Throwable t2) { - throw new IgnoreMeException(); - } + dropView(globalState, view.getName()); } } } + + private void dropView(G globalState, String viewName) { + try { + globalState.executeStatement(new SQLQueryAdapter("DROP VIEW " + viewName, true)); + } catch (Throwable t2) { + throw new IgnoreMeException(); + } + } } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 8fd79cb99..66a5e5d7d 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -246,8 +246,6 @@ private static void addArrayErrors(ExpectedErrors errors) { errors.add("unimplemented: nested arrays not supported"); // e.g., casting a string {{1}} to an array errors.add("malformed array"); - // errors.add("https://github.com/cockroachdb/cockroach/issues/35707"); // arrays don't support ORDER BY - errors.add("as bytes[], found type: varbit[]"); errors.add("to be of type decimal[], found type float[]"); errors.add("to be of type int[], found type decimal[]"); diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 1c1dd3531..c245f4cce 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -293,7 +293,7 @@ public static CockroachDBSchema fromConnection(SQLConnection con, String databas for (CockroachDBColumn c : databaseColumns) { c.setTable(t); } - // To avoid some situations that columns can not be got. + // To avoid some situations that columns can not be retrieved. if (databaseColumns.isEmpty()) { continue; } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index cef9862de..42740506d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -15,14 +15,9 @@ private CockroachDBSetClusterSettingGenerator() { // https://www.cockroachlabs.com/docs/stable/set-vars.html private enum CockroachDBClusterSetting { - // BUFFER_INCREMENT("kv.bulk_ingest.buffer_increment", (g) -> "'" + Randomly.getUncachedDouble() + "'"), BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), - // SQL_DEFAULTS_VECTORIZE_ROW_COUNT_THRESHOLD("sql.defaults.vectorize_row_count_threshold", - // (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), - // SQL_DEFAULTS_EXPERIMENTAL_OPTIMIZER_FOREIGN_KEYS_ENABLED("sql.defaults.experimental_optimizer_foreign_keys.enabled", - // CockroachDBSetSessionGenerator::onOff), SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), SQL_STATS_HISTOGRAM_COLLECTION_ENABLED("sql.stats.histogram_collection.enabled", CockroachDBSetSessionGenerator::onOff); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java index e99eaaa27..0ade4a3ef 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetSessionGenerator.java @@ -25,9 +25,7 @@ private enum CockroachDBSetting { ENABLE_IMPLICIT_SELECT_FOR_UPDATE(CockroachDBSetSessionGenerator::onOff), ENABLE_INSERT_FAST_PATH(CockroachDBSetSessionGenerator::onOff), ENABLE_ZIGZAG_JOIN(CockroachDBSetSessionGenerator::onOff), - // EXPERIMENTAL_ENABLE_HASH_SHARDED_INDEXES(CockroachDBSetSessionGenerator::onOff), SERIAL_NORMALIZATION((g) -> Randomly.fromOptions("'rowid'", "'virtual_sequence'")), - // EXTRA_FLOAT_DIGITS((g) -> g.getRandomly().getInteger(-15, 3)), REORDER_JOINS_LIMIT((g) -> g.getRandomly().getInteger(0, Integer.MAX_VALUE)), // SQL_SAFE_UPDATES((g) -> "off"), TRACING(CockroachDBSetSessionGenerator::onOff), /* diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index d10000352..904ee3a9f 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -15,8 +15,6 @@ public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add("is referenced by foreign key"); - // https://github.com/cockroachdb/cockroach/issues/47030 - // errors.add("unexpected value: "); StringBuilder sb = new StringBuilder(); sb.append("TRUNCATE"); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index c2e6706cc..01fcc8539 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -96,9 +96,17 @@ public String getUnterminatedQueryString() { } public void checkException(Exception e) throws AssertionError { - if (!expectedErrors.errorIsExpected(e.getMessage())) { - throw new AssertionError(query, e); + Throwable ex = e; + + while (ex != null) { + if (expectedErrors.errorIsExpected(ex.getMessage())) { + return; + } else { + ex = ex.getCause(); + } } + + throw new AssertionError(query, e); } @Override diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 95979264c..f37bdb685 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -21,19 +21,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { // functions errors.add("BIGINT value is out of range"); errors.add("doesn't have a default value"); // default - errors.add("is not valid for CHARACTER SET"); - - // known issue: https://github.com/pingcap/tidb/issues/14819 - // errors.add("Wrong plan type for dataReaderBuilder"); - errors.add("DOUBLE value is out of range"); - // errors.add("index out of range"); // https://github.com/pingcap/tidb/issues/15810 - // errors.add("baseBuiltinFunc.evalString() should never be called, please contact the TiDB team for help"); // - // https://github.com/pingcap/tidb/issues/15847 - // errors.add("unsupport column type for encode 6"); // https://github.com/pingcap/tidb/issues/15850 - errors.add("Data truncation: %s value is out of range in '%s'"); errors.add("Truncated incorrect FLOAT value"); errors.add("Bad Number"); @@ -47,13 +37,6 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Can\'t find column"); // https://github.com/pingcap/tidb/issues/35527 errors.add("Cannot convert"); // https://github.com/pingcap/tidb/issues/35652 - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/57 - // errors.add("For input string: \"+Inf\""); - - // errors.add("inconsistent index"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/58 - - // errors.add("Illegal mix of collations"); - if (TiDBBugs.bug35677) { errors.add("for function inet_aton"); } @@ -92,11 +75,6 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Incorrect decimal value"); errors.add("error parsing regexp"); errors.add("is not valid for CHARACTER SET"); - - // if (true) { - // // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/54 - // errors.add("Miss column"); - // } } } diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index b147ab8ab..51bdd6072 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -334,8 +334,7 @@ private static List getTableColumns(SQLConnection con, String tableN columns.add(c); } } - } catch (SQLException e) { - + } catch (SQLException e) { // Happens when } return columns; } diff --git a/src/sqlancer/tidb/ast/TiDBFunctionCall.java b/src/sqlancer/tidb/ast/TiDBFunctionCall.java index 1aaa87827..c7bdb2d99 100644 --- a/src/sqlancer/tidb/ast/TiDBFunctionCall.java +++ b/src/sqlancer/tidb/ast/TiDBFunctionCall.java @@ -92,7 +92,7 @@ public int getNrArgs() { // [...] REPLACE(3), // REVERSE(1), // - // RIGHT(2), // https://github.com/pingcap/tidb/issues/35623 + RIGHT(2), // // RPAD TODO RTRIM(1), // SPACE(1), // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/6 diff --git a/src/sqlancer/tidb/ast/TiDBRegexOperation.java b/src/sqlancer/tidb/ast/TiDBRegexOperation.java index c8da018d8..f5f011dd6 100644 --- a/src/sqlancer/tidb/ast/TiDBRegexOperation.java +++ b/src/sqlancer/tidb/ast/TiDBRegexOperation.java @@ -9,9 +9,8 @@ public class TiDBRegexOperation extends BinaryOperatorNode actions = Randomly.nonEmptySubset(Action.values()); for (Action a : actions) { sb.append(" "); - * switch (a) { case AUTO_INCREMENT: sb.append("AUTO_INCREMENT="); sb.append(Randomly.getNotCachedInteger(0, - * Integer.MAX_VALUE)); break; case PRE_SPLIT_REGIONS: sb.append("PRE_SPLIT_REGIONS="); - * sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); break; case SHARD_ROW_ID_BITS: - * sb.append("SHARD_ROW_ID_BITS="); sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); - * errors.add("Unsupported shard_row_id_bits for table with primary key as row id"); break; default: throw new - * AssertionError(a); } } - */ } private boolean canUseAsUnique(TiDBCompositeDataType type) { @@ -148,9 +138,6 @@ private boolean canUseAsUnique(TiDBCompositeDataType type) { } private void appendType(StringBuilder sb, TiDBCompositeDataType type) { - // if (type.getPrimitiveDataType() == TiDBDataType.CHAR) { - // throw new IgnoreMeException(); - // } sb.append(type.toString()); appendSpecifiers(sb, type.getPrimitiveDataType()); appendSizeSpecifiers(sb, type.getPrimitiveDataType()); diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 0c2459b28..4912899fe 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -43,7 +43,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); sb.append(TiDBVisitor.asString(gen.generateExpression())); - // errors.add("Data Too Long"); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/43 } TiDBErrors.addInsertErrors(errors); From f99004f74cd938b14747706384b7ccf1a7645cfb Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Wed, 5 Oct 2022 16:08:44 +0800 Subject: [PATCH 190/856] Bypass errors in Alter --- src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index fb0d42892..057bb2789 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -31,6 +31,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("charset"); errors.add("not supported"); errors.add("SQL syntax"); + errors.add("can't drop"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); From d57fdbfee07f43556f8bc6d08b009cb46a554884 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Wed, 5 Oct 2022 16:50:28 +0800 Subject: [PATCH 191/856] support datetime in TiDB --- src/sqlancer/tidb/TiDBSchema.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 51bdd6072..7caff2974 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -256,6 +256,13 @@ private static TiDBCompositeDataType getColumnType(String typeString) { case "tinyblob": primitiveType = TiDBDataType.BLOB; break; + case "date": + case "datetime": + case "timestamp": + case "time": + case "year": + primitiveType = TiDBDataType.NUMERIC; + break; default: throw new AssertionError(trimmedStringType); } From 8b8f4e4853edb9e30c10ff3b578fb32f5a975b48 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 13:05:14 +0400 Subject: [PATCH 192/856] Minor review fixes --- src/sqlancer/yugabyte/YugabyteBugs.java | 6 ++++ src/sqlancer/yugabyte/ycql/YCQLProvider.java | 11 +++--- src/sqlancer/yugabyte/ycql/YCQLSchema.java | 7 +--- src/sqlancer/yugabyte/ysql/YSQLProvider.java | 29 +++++++-------- .../ysql/gen/YSQLAlterTableGenerator.java | 4 +-- .../yugabyte/ysql/gen/YSQLCommon.java | 23 ++---------- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 15 ++------ .../ysql/gen/YSQLTruncateGenerator.java | 12 ------- .../ysql/gen/YSQLVacuumGenerator.java | 5 +-- .../yugabyte/ysql/gen/YSQLViewGenerator.java | 14 -------- .../yugabyte/ysql/oracle/YSQLCatalog.java | 8 +++-- .../yugabyte/ysql/oracle/YSQLFuzzer.java | 36 +++---------------- 12 files changed, 45 insertions(+), 125 deletions(-) create mode 100644 src/sqlancer/yugabyte/YugabyteBugs.java diff --git a/src/sqlancer/yugabyte/YugabyteBugs.java b/src/sqlancer/yugabyte/YugabyteBugs.java new file mode 100644 index 000000000..1e944e0e2 --- /dev/null +++ b/src/sqlancer/yugabyte/YugabyteBugs.java @@ -0,0 +1,6 @@ +package sqlancer.yugabyte; + +public interface YugabyteBugs { + // https://github.com/yugabyte/yugabyte-db/issues/11357 + boolean bug11357 = true; +} diff --git a/src/sqlancer/yugabyte/ycql/YCQLProvider.java b/src/sqlancer/yugabyte/ycql/YCQLProvider.java index 3d408f8c1..8bc8cbf84 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLProvider.java +++ b/src/sqlancer/yugabyte/ycql/YCQLProvider.java @@ -122,9 +122,10 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep e.printStackTrace(); } final String host = globalState.getOptions().getHost(); - final String url = "jdbc:cassandra://%s:9042/%s?localdatacenter=%s"; + final int port = globalState.getOptions().getPort(); + final String url = "jdbc:cassandra://%s:%s/%s?localdatacenter=%s"; final Connection connection = DriverManager.getConnection( - String.format(url, host, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); + String.format(url, host, port, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); try (Statement stmt = connection.createStatement()) { try { @@ -132,7 +133,7 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep } catch (Exception se) { // try again List tableNames = getTableNames( - new SQLConnection(DriverManager.getConnection(String.format(url, host, + new SQLConnection(DriverManager.getConnection(String.format(url, host, port, globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))), globalState.getDatabaseName()); for (String tableName : tableNames) { @@ -144,8 +145,8 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep stmt.execute("CREATE KEYSPACE IF NOT EXISTS " + globalState.getDatabaseName()); } - return new SQLConnection(DriverManager.getConnection(String.format(url, host, globalState.getDatabaseName(), - globalState.getDbmsSpecificOptions().datacenter))); + return new SQLConnection(DriverManager.getConnection(String.format(url, host, port, + globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))); } @Override diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java index 4845520d2..1534b9b56 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLSchema.java +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -7,7 +7,6 @@ import java.util.Collections; import java.util.List; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.DBMSCommon; @@ -195,12 +194,8 @@ private static YCQLCompositeDataType getColumnType(String typeString) { case "TIMESTAMP": primitiveType = YCQLDataType.TIMESTAMP; break; - case "INTERVAL": - throw new IgnoreMeException(); - // TODO: caused when a view contains a computation like ((TIMESTAMP '1970-01-05 11:26:57')-(TIMESTAMP - // '1969-12-29 06:50:27')) default: - throw new IgnoreMeException(); + throw new AssertionError(); } return new YCQLCompositeDataType(primitiveType, size); } diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java index 278e5316e..f70b042d9 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLProvider.java +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -22,7 +22,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.yugabyte.ysql.YSQLOptions.YSQLOracleFactory; +import sqlancer.yugabyte.YugabyteBugs; import sqlancer.yugabyte.ysql.gen.YSQLAlterTableGenerator; import sqlancer.yugabyte.ysql.gen.YSQLAnalyzeGenerator; import sqlancer.yugabyte.ysql.gen.YSQLCommentGenerator; @@ -45,7 +45,8 @@ @AutoService(DatabaseProvider.class) public class YSQLProvider extends SQLProviderAdapter { - public static final Object CREATION_LOCK = new Object(); + // TODO Due to yugabyte problems with parallel DDL we need this lock object + public static final Object DDL_LOCK = new Object(); /** * Generate only data types and expressions that are understood by PQS. */ @@ -132,11 +133,6 @@ public void generateDatabase(YSQLGlobalState globalState) throws Exception { @Override public SQLConnection createDatabase(YSQLGlobalState globalState) throws SQLException { - if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() - .anyMatch((o) -> o == YSQLOracleFactory.PQS)) { - generateOnlyKnown = true; - } - username = globalState.getOptions().getUserName(); password = globalState.getOptions().getPassword(); host = globalState.getOptions().getHost(); @@ -197,7 +193,7 @@ public String getDBMSName() { // for some reason yugabyte unable to create few databases simultaneously private void createDatabaseSync(YSQLGlobalState globalState, String entryDatabaseName) throws SQLException { - synchronized (CREATION_LOCK) { + synchronized (DDL_LOCK) { exceptionLessSleep(5000); Connection con = createConnectionSafely(entryURL, username, password); @@ -246,7 +242,7 @@ protected void readFunctions(YSQLGlobalState globalState) throws SQLException { } protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { - synchronized (CREATION_LOCK) { + synchronized (DDL_LOCK) { boolean prevCreationFailed = false; // small optimization - wait only after failed requests while (globalState.getSchema().getDatabaseTables().size() < numTables) { if (!prevCreationFailed) { @@ -270,7 +266,7 @@ private void exceptionLessSleep(long timeout) { try { Thread.sleep(timeout); } catch (InterruptedException e) { - e.printStackTrace(); + throw new AssertionError(); } } @@ -296,10 +292,15 @@ private String getCreateDatabaseCommand(YSQLGlobalState state) { sb.append(Randomly.fromOptions("utf8")); sb.append("' "); } - // disabled due to https://github.com/yugabyte/yugabyte-db/issues/11357 - // if (Randomly.getBoolean()) { - // sb.append("COLOCATED = true "); - // } + + if (Randomly.getBoolean()) { + if (YugabyteBugs.bug11357) { + throw new IgnoreMeException(); + } + + sb.append("COLOCATED = true "); + } + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java index 035b2fbe4..d489b169e 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java @@ -99,7 +99,7 @@ public SQLQueryAdapter generate() { break; case ADD_TABLE_CONSTRAINT: sb.append("ADD "); - sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); + sb.append("CONSTRAINT ").append(r.getAlphabeticChar()).append(" "); YSQLCommon.addTableConstraint(sb, randomTable, globalState, errors); errors.add("already exists"); errors.add("multiple primary keys for table"); @@ -130,7 +130,7 @@ public SQLQueryAdapter generate() { break; case ADD_TABLE_CONSTRAINT_USING_INDEX: sb.append("ADD "); - sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); + sb.append("CONSTRAINT ").append(r.getAlphabeticChar()).append(" "); sb.append(Randomly.fromOptions("UNIQUE", "PRIMARY KEY")); sb.append(" USING INDEX "); sb.append(randomTable.getRandomIndex().getIndexName()); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java index e92af3e0f..89f3dd3a8 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java @@ -8,6 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.YugabyteBugs; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; @@ -213,27 +214,7 @@ public static boolean appendDataType(YSQLDataType type, StringBuilder sb, boolea public static void generateWith(StringBuilder sb, YSQLGlobalState globalState, ExpectedErrors errors, List columnsToBeAdded, boolean isTemporaryTable) { if (Randomly.getBoolean()) { - if (Randomly.getBoolean()) { - sb.append(" "); - // disabled https://github.com/yugabyte/yugabyte-db/issues/11357 - // sb.append(" WITH ("); - // ArrayList values = new ArrayList<>(Arrays.asList(StorageParameters.values())); - // errors.add("unrecognized parameter"); - // errors.add("ALTER TABLE / ADD CONSTRAINT USING INDEX is not supported on partitioned tables"); - // List subset = Randomly.nonEmptySubset(values); - // int i = 0; - // for (StorageParameters parameter : subset) { - // if (i++ != 0) { - // sb.append(", "); - // } - // sb.append(parameter.parameter); - // sb.append("="); - // sb.append(parameter.op.apply(globalState.getRandomly())); - // } - // sb.append(")"); - } else { - sb.append(" WITHOUT OIDS "); - } + sb.append(" WITHOUT OIDS "); } else if (Randomly.getBoolean() && !isTemporaryTable) { if (Randomly.getBoolean()) { sb.append(" SPLIT INTO "); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index faa7908b0..9f814025d 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -28,13 +29,6 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { sb.append(" UNIQUE"); } sb.append(" INDEX "); - /* - * Commented out as a workaround for https://www.postgresql.org/message-id/CA%2Bu7OA4XYhc- - * qyCgJqwwgMGZDWAyeH821oa5oMzm_HEifZ4BeA%40mail.gmail.com - */ - // if (Randomly.getBoolean()) { - // sb.append("CONCURRENTLY "); - // } YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized // views String indexName = getNewIndexName(randomTable); @@ -71,11 +65,6 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { sb.append(")"); } - // if (Randomly.getBoolean()) { - // sb.append(" "); - // sb.append("COLLATE "); - // sb.append(Randomly.fromOptions("C", "POSIX")); - // } if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" "); sb.append(globalState.getRandomOpclass()); @@ -97,7 +86,7 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { if (Randomly.getBoolean() && method != IndexType.HASH) { sb.append(" INCLUDE("); List columns = randomTable.getRandomNonEmptyColumnSubset(); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); sb.append(")"); } if (Randomly.getBoolean()) { diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java index d10bccee6..449654d99 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTruncateGenerator.java @@ -19,21 +19,9 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { if (Randomly.getBoolean()) { sb.append(" TABLE"); } - // TODO partitions - // if (Randomly.getBoolean()) { - // sb.append(" ONLY"); - // } sb.append(" "); sb.append(globalState.getSchema().getDatabaseTablesRandomSubsetNotEmpty().stream().map(AbstractTable::getName) .collect(Collectors.joining(", "))); - // if (Randomly.getBoolean()) { - // sb.append(" "); - // sb.append(Randomly.fromOptions("RESTART IDENTITY", "CONTINUE IDENTITY")); - // } - // if (Randomly.getBoolean()) { - // sb.append(" "); - // sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); - // } return new SQLQueryAdapter(sb.toString(), ExpectedErrors .from("cannot truncate a table referenced in a foreign key constraint", "is not a table")); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java index f873f7812..9efe027b1 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLVacuumGenerator.java @@ -10,10 +10,7 @@ private YSQLVacuumGenerator() { } public static SQLQueryAdapter create(YSQLGlobalState globalState) { - String sb = "VACUUM"; - ExpectedErrors errors = new ExpectedErrors(); - errors.add("VACUUM cannot run inside a transaction block"); - return new SQLQueryAdapter(sb, errors); + return new SQLQueryAdapter("VACUUM", ExpectedErrors.from("VACUUM cannot run inside a transaction block")); } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java index a5b57fbd8..a34be987b 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -16,11 +16,8 @@ private YSQLViewGenerator() { public static SQLQueryAdapter create(YSQLGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE"); - // boolean materialized; - // boolean recursive = false; if (Randomly.getBoolean()) { sb.append(" MATERIALIZED"); - // materialized = true; } else { if (Randomly.getBoolean()) { sb.append(" OR REPLACE"); @@ -28,11 +25,6 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { if (Randomly.getBoolean()) { sb.append(Randomly.fromOptions(" TEMP", " TEMPORARY")); } - // if (Randomly.getBoolean()) { - // sb.append(" RECURSIVE"); - // recursive = true; - // } - // materialized = false; } sb.append(" VIEW "); int i = 0; @@ -54,12 +46,6 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { sb.append(DBMSCommon.createColumnName(i)); } sb.append(")"); - // if (Randomly.getBoolean() && !materialized && !recursive) { - // sb.append(" WITH "); - // sb.append(Randomly.fromOptions("CASCADED", "LOCAL")); - // sb.append(" CHECK OPTION"); - // errors.add("WITH CHECK OPTION is supported only on automatically updatable views"); - // } sb.append(" AS ("); YSQLSelect select = YSQLRandomQueryGenerator.createRandomQuery(nrColumns, globalState); sb.append(YSQLVisitor.asString(select)); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java index a873ce505..1eb6b9ff4 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java @@ -1,6 +1,6 @@ package sqlancer.yugabyte.ysql.oracle; -import static sqlancer.yugabyte.ysql.YSQLProvider.CREATION_LOCK; +import static sqlancer.yugabyte.ysql.YSQLProvider.DDL_LOCK; import java.util.Arrays; import java.util.List; @@ -49,12 +49,14 @@ private YSQLProvider.Action getRandomAction(List actions) { } protected void createTables(YSQLGlobalState globalState, int numTables) throws Exception { - synchronized (CREATION_LOCK) { + synchronized (DDL_LOCK) { while (globalState.getSchema().getDatabaseTables().size() < numTables) { + // TODO concurrent DDLs may produce a lot of noise in test logs so its disabled right now + // added timeout to avoid possible catalog collisions try { Thread.sleep(1000); } catch (InterruptedException e) { - e.printStackTrace(); + throw new AssertionError(); } try { diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java index e8f791b3f..a5af7111f 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java @@ -2,11 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Locale; -import java.util.Map; - -import com.typesafe.config.Config; -import com.typesafe.config.ConfigFactory; import sqlancer.Randomly; import sqlancer.common.oracle.TestOracle; @@ -40,33 +35,12 @@ public YSQLFuzzer(YSQLGlobalState globalState) { errors.add("Connection refused"); errors.add("Connection to"); - // get config from -Dconfig.file="path/to/fuzzer.conf" testQueries = new ArrayList<>(); - try { - Config config = ConfigFactory.load(); - ArrayList queriesList = (ArrayList) config.getList("queries").unwrapped(); - for (Object configValue : queriesList) { - String type = ((String) ((Map) configValue).get("type")).toUpperCase(Locale.ROOT); - Integer weight = (Integer) ((Map) configValue).get("weight"); - - Query query = type.equalsIgnoreCase("SELECT") ? new SelectQuery() - : new ActionQuery(YSQLProvider.Action.valueOf(type)); - - for (int i = 0; i < weight; i++) { - testQueries.add(query); - } - } - } catch (Exception e) { - // do nothing - } finally { - if (testQueries.isEmpty()) { - System.out.println("No configuration found. Using just random select statements"); - testQueries.add(new SelectQuery()); - testQueries.add(new ActionQuery(YSQLProvider.Action.UPDATE)); - testQueries.add(new ActionQuery(YSQLProvider.Action.DELETE)); - testQueries.add(new ActionQuery(YSQLProvider.Action.INSERT)); - } - } + + testQueries.add(new SelectQuery()); + testQueries.add(new ActionQuery(YSQLProvider.Action.UPDATE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.DELETE)); + testQueries.add(new ActionQuery(YSQLProvider.Action.INSERT)); } @Override From 4c3304135c1846d42b13f08f4d900638a0a78184 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 14:49:45 +0400 Subject: [PATCH 193/856] Fixed README, CI Added Test and YSQLErrors --- .github/workflows/main.yml | 21 +++ README.md | 27 ++-- src/sqlancer/yugabyte/YugabyteBugs.java | 9 +- src/sqlancer/yugabyte/ycql/YCQLOptions.java | 9 +- src/sqlancer/yugabyte/ycql/YCQLProvider.java | 16 +- src/sqlancer/yugabyte/ysql/YSQLErrors.java | 151 ++++++++++++++++++ src/sqlancer/yugabyte/ysql/YSQLProvider.java | 14 +- .../ysql/gen/YSQLAlterTableGenerator.java | 7 +- .../yugabyte/ysql/gen/YSQLCommon.java | 145 +---------------- .../ysql/gen/YSQLDeleteGenerator.java | 3 +- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 3 +- .../ysql/gen/YSQLInsertGenerator.java | 7 +- .../yugabyte/ysql/gen/YSQLTableGenerator.java | 7 +- .../ysql/gen/YSQLUpdateGenerator.java | 5 +- .../yugabyte/ysql/gen/YSQLViewGenerator.java | 7 +- .../yugabyte/ysql/oracle/YSQLCatalog.java | 6 +- .../yugabyte/ysql/oracle/YSQLFuzzer.java | 10 +- .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 6 +- .../YSQLPivotedQuerySynthesisOracle.java | 6 +- .../oracle/tlp/YSQLTLPAggregateOracle.java | 4 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 6 +- .../ysql/oracle/tlp/YSQLTLPHavingOracle.java | 4 +- test/sqlancer/dbms/TestYugabyte.java | 29 ++++ 23 files changed, 298 insertions(+), 204 deletions(-) create mode 100644 src/sqlancer/yugabyte/ysql/YSQLErrors.java create mode 100644 test/sqlancer/dbms/TestYugabyte.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bfbbe60cc..7bf958bc1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -297,6 +297,27 @@ jobs: - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test + yugabyte: + name: DBMS Tests (Yugabyte) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Setup Yugabyte + run: | + docker pull yugabytedb/yugabyte:latest + docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false + sleep 5 + - name: Run Tests + run: YUGABYTE_AVAILABLE=true mvn -Dtest=TestYugabyte test + java13: name: Java 13 Compatibility (DuckDB) runs-on: ubuntu-latest diff --git a/README.md b/README.md index 2209e7589..d163f049c 100644 --- a/README.md +++ b/README.md @@ -47,19 +47,20 @@ Please find the `.bib` entries [here](docs/PAPERS.md). Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. -| DBMS | Status | Expression Generation | Description | -|-------------|-------------|-----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | -| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | -| PostgreSQL | Working | Typed | | -| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | -| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | -| CockroachDB | Working | Typed | | -| TiDB | Working | Untyped | | -| DuckDB | Working | Untyped, Generic | | -| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | -| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | -| OceanBase | Working | Untyped | | +| DBMS | Status | Expression Generation | Description | +|------------------------------|-------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | +| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | +| PostgreSQL | Working | Typed | | +| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | +| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | +| CockroachDB | Working | Typed | | +| TiDB | Working | Untyped | | +| DuckDB | Working | Untyped, Generic | | +| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | +| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | +| OceanBase | Working | Untyped | | +| YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | # Using SQLancer diff --git a/src/sqlancer/yugabyte/YugabyteBugs.java b/src/sqlancer/yugabyte/YugabyteBugs.java index 1e944e0e2..424ba3217 100644 --- a/src/sqlancer/yugabyte/YugabyteBugs.java +++ b/src/sqlancer/yugabyte/YugabyteBugs.java @@ -1,6 +1,11 @@ package sqlancer.yugabyte; -public interface YugabyteBugs { +public final class YugabyteBugs { + // https://github.com/yugabyte/yugabyte-db/issues/11357 - boolean bug11357 = true; + public static boolean bug11357 = true; + + private YugabyteBugs() { + } + } diff --git a/src/sqlancer/yugabyte/ycql/YCQLOptions.java b/src/sqlancer/yugabyte/ycql/YCQLOptions.java index 5955d28dd..7e35e2cea 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLOptions.java +++ b/src/sqlancer/yugabyte/ycql/YCQLOptions.java @@ -14,9 +14,14 @@ import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.test.YCQLFuzzer; -@Parameters(commandDescription = "YCQL") +@Parameters(separators = "=", commandDescription = "YCQL (default port: " + YCQLOptions.DEFAULT_PORT + + ", default host: " + YCQLOptions.DEFAULT_HOST) public class YCQLOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 9042; + public static final String DEFAULT_DATACENTER = "datacenter1"; + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) public int maxNumDeletes = 1; @@ -24,7 +29,7 @@ public class YCQLOptions implements DBMSSpecificOptions { public int maxNumUpdates = 5; @Parameter(names = "--datacenter", description = "YCQL datacenter, can be found in system.local table", arity = 1) - public String datacenter = "datacenter1"; + public String datacenter = DEFAULT_DATACENTER; @Parameter(names = "--oracle") public List oracles = Arrays.asList(YCQLOracleFactory.FUZZER); diff --git a/src/sqlancer/yugabyte/ycql/YCQLProvider.java b/src/sqlancer/yugabyte/ycql/YCQLProvider.java index 8bc8cbf84..59624f588 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLProvider.java +++ b/src/sqlancer/yugabyte/ycql/YCQLProvider.java @@ -13,6 +13,7 @@ import sqlancer.AbstractAction; import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; @@ -119,10 +120,19 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep try { Class.forName("com.ing.data.cassandra.jdbc.CassandraDriver"); } catch (ClassNotFoundException e) { - e.printStackTrace(); + throw new AssertionError(); } - final String host = globalState.getOptions().getHost(); - final int port = globalState.getOptions().getPort(); + + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + + if (host == null) { + host = YCQLOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = YCQLOptions.DEFAULT_PORT; + } + final String url = "jdbc:cassandra://%s:%s/%s?localdatacenter=%s"; final Connection connection = DriverManager.getConnection( String.format(url, host, port, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); diff --git a/src/sqlancer/yugabyte/ysql/YSQLErrors.java b/src/sqlancer/yugabyte/ysql/YSQLErrors.java new file mode 100644 index 000000000..a2cee1add --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLErrors.java @@ -0,0 +1,151 @@ +package sqlancer.yugabyte.ysql; + +import sqlancer.common.query.ExpectedErrors; + +public final class YSQLErrors { + + private YSQLErrors() { + } + + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.add("An I/O error occurred while sending to the backend"); + errors.add("Conflicts with committed transaction"); + errors.add("cannot be changed"); + errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); + + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); + errors.add("but it cannot be referenced from this part of the query"); + errors.add("missing FROM-clause entry for table"); + + errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); + } + + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.add("PRIMARY KEY containing column of type 'INET' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'VARBIT' not yet supported"); + errors.add("PRIMARY KEY containing column of type 'INT4RANGE' not yet supported"); + errors.add("INDEX on column of type 'INET' not yet supported"); + errors.add("INDEX on column of type 'VARBIT' not yet supported"); + errors.add("INDEX on column of type 'INT4RANGE' not yet supported"); + errors.add("is not commutative"); // exclude + errors.add("cannot be changed"); + errors.add("operator requires run-time type coercion"); // exclude + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.add("syntax error at or near \"(\""); + errors.add("does not exist"); + errors.add("is not unique"); + errors.add("cannot be changed"); + errors.add("invalid reference to FROM-clause entry for table"); + + errors.add("Invalid column number"); + errors.add("specified more than once"); + errors.add("You might need to add explicit type casts"); + errors.add("invalid regular expression"); + errors.add("could not determine which collation to use"); + errors.add("invalid input syntax for integer"); + errors.add("invalid regular expression"); + errors.add("operator does not exist"); + errors.add("quantifier operand invalid"); + errors.add("collation mismatch"); + errors.add("collations are not supported"); + errors.add("operator is not unique"); + errors.add("is not a valid binary digit"); + errors.add("invalid hexadecimal digit"); + errors.add("invalid hexadecimal data: odd number of digits"); + errors.add("zero raised to a negative power is undefined"); + errors.add("cannot convert infinity to numeric"); + errors.add("division by zero"); + errors.add("invalid input syntax for type money"); + errors.add("invalid input syntax for type"); + errors.add("cannot cast type"); + errors.add("value overflows numeric format"); + errors.add("is of type boolean but expression is of type text"); + errors.add("a negative number raised to a non-integer power yields a complex result"); + errors.add("could not determine polymorphic type because input has type unknown"); + + addToCharFunctionErrors(errors); + addBitStringOperationErrors(errors); + addFunctionErrors(errors); + addCommonRangeExpressionErrors(errors); + addCommonRegexExpressionErrors(errors); + } + + public static void addToCharFunctionErrors(ExpectedErrors errors) { + errors.add("multiple decimal points"); + errors.add("and decimal point together"); + errors.add("multiple decimal points"); + errors.add("cannot use \"S\" twice"); + errors.add("must be ahead of \"PR\""); + errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); + errors.add("cannot use \"S\" and \"SG\" together"); + errors.add("cannot use \"S\" and \"MI\" together"); + errors.add("cannot use \"S\" and \"PL\" together"); + errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); + errors.add("is not a number"); + } + + public static void addBitStringOperationErrors(ExpectedErrors errors) { + errors.add("cannot XOR bit strings of different sizes"); + errors.add("cannot AND bit strings of different sizes"); + errors.add("cannot OR bit strings of different sizes"); + errors.add("must be type boolean, not type text"); + } + + public static void addFunctionErrors(ExpectedErrors errors) { + errors.add("out of valid range"); // get_bit/get_byte + errors.add("cannot take logarithm of a negative number"); + errors.add("cannot take logarithm of zero"); + errors.add("requested character too large for encoding"); // chr + errors.add("null character not permitted"); // chr + errors.add("requested character not valid for encoding"); // chr + errors.add("requested length too large"); // repeat + errors.add("invalid memory alloc request size"); // repeat + errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + } + + public static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + errors.add("is not a valid hexadecimal digit"); + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.add("range lower bound must be less than or equal to range upper bound"); + errors.add("result of range difference would not be contiguous"); + errors.add("out of range"); + errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.add("value too long for type character"); + errors.add("not found in view targetlist"); + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in GROUP BY"); + } + + public static void addViewErrors(ExpectedErrors errors) { + errors.add("already exists"); + errors.add("cannot drop columns from view"); + errors.add("non-integer constant in ORDER BY"); // TODO + errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO + errors.add("cannot change data type of view column"); + errors.add("specified more than once"); // TODO + errors.add("materialized views must not use temporary tables or views"); + errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); + errors.add("is not a view"); + errors.add("non-integer constant in DISTINCT ON"); + errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java index f70b042d9..e72790344 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLProvider.java +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -22,7 +22,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.yugabyte.YugabyteBugs; import sqlancer.yugabyte.ysql.gen.YSQLAlterTableGenerator; import sqlancer.yugabyte.ysql.gen.YSQLAnalyzeGenerator; import sqlancer.yugabyte.ysql.gen.YSQLCommentGenerator; @@ -142,6 +141,13 @@ public SQLConnection createDatabase(YSQLGlobalState globalState) throws SQLExcep String entryDatabaseName = entryPath.substring(1); databaseName = globalState.getDatabaseName(); + if (host == null) { + host = YSQLOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = YSQLOptions.DEFAULT_PORT; + } + try { URI uri = new URI(entryURL); String userInfoURI = uri.getUserInfo(); @@ -294,9 +300,9 @@ private String getCreateDatabaseCommand(YSQLGlobalState state) { } if (Randomly.getBoolean()) { - if (YugabyteBugs.bug11357) { - throw new IgnoreMeException(); - } + // if (YugabyteBugs.bug11357) { + // throw new IgnoreMeException(); + // } sb.append("COLOCATED = true "); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java index d489b169e..c4a4effd8 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLAlterTableGenerator.java @@ -6,6 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; @@ -26,9 +27,9 @@ public static SQLQueryAdapter create(YSQLTable randomTable, YSQLGlobalState glob } public List getActions(ExpectedErrors errors) { - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonInsertUpdateErrors(errors); - YSQLCommon.addCommonTableErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonInsertUpdateErrors(errors); + YSQLErrors.addCommonTableErrors(errors); errors.add("duplicate key value violates unique constraint"); errors.add("cannot drop key column"); errors.add("cannot drop desired object(s) because other objects depend on them"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java index 89f3dd3a8..09b559e4c 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java @@ -8,7 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.yugabyte.YugabyteBugs; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; @@ -22,126 +22,6 @@ public final class YSQLCommon { private YSQLCommon() { } - public static void addCommonFetchErrors(ExpectedErrors errors) { - errors.add("An I/O error occurred while sending to the backend"); - errors.add("Conflicts with committed transaction"); - errors.add("cannot be changed"); - errors.add("SET TRANSACTION ISOLATION LEVEL must be called before any query"); - - errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); - errors.add("but it cannot be referenced from this part of the query"); - errors.add("missing FROM-clause entry for table"); - - errors.add("canceling statement due to statement timeout"); - - errors.add("non-integer constant in"); - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("GROUP BY position"); - } - - public static void addCommonTableErrors(ExpectedErrors errors) { - errors.add("PRIMARY KEY containing column of type 'INET' not yet supported"); - errors.add("PRIMARY KEY containing column of type 'VARBIT' not yet supported"); - errors.add("PRIMARY KEY containing column of type 'INT4RANGE' not yet supported"); - errors.add("INDEX on column of type 'INET' not yet supported"); - errors.add("INDEX on column of type 'VARBIT' not yet supported"); - errors.add("INDEX on column of type 'INT4RANGE' not yet supported"); - errors.add("is not commutative"); // exclude - errors.add("cannot be changed"); - errors.add("operator requires run-time type coercion"); // exclude - } - - public static void addCommonExpressionErrors(ExpectedErrors errors) { - errors.add("syntax error at or near \"(\""); - errors.add("does not exist"); - errors.add("is not unique"); - errors.add("cannot be changed"); - errors.add("invalid reference to FROM-clause entry for table"); - - errors.add("Invalid column number"); - errors.add("specified more than once"); - errors.add("You might need to add explicit type casts"); - errors.add("invalid regular expression"); - errors.add("could not determine which collation to use"); - errors.add("invalid input syntax for integer"); - errors.add("invalid regular expression"); - errors.add("operator does not exist"); - errors.add("quantifier operand invalid"); - errors.add("collation mismatch"); - errors.add("collations are not supported"); - errors.add("operator is not unique"); - errors.add("is not a valid binary digit"); - errors.add("invalid hexadecimal digit"); - errors.add("invalid hexadecimal data: odd number of digits"); - errors.add("zero raised to a negative power is undefined"); - errors.add("cannot convert infinity to numeric"); - errors.add("division by zero"); - errors.add("invalid input syntax for type money"); - errors.add("invalid input syntax for type"); - errors.add("cannot cast type"); - errors.add("value overflows numeric format"); - errors.add("is of type boolean but expression is of type text"); - errors.add("a negative number raised to a non-integer power yields a complex result"); - errors.add("could not determine polymorphic type because input has type unknown"); - addToCharFunctionErrors(errors); - addBitStringOperationErrors(errors); - addFunctionErrors(errors); - addCommonRangeExpressionErrors(errors); - addCommonRegexExpressionErrors(errors); - } - - private static void addToCharFunctionErrors(ExpectedErrors errors) { - errors.add("multiple decimal points"); - errors.add("and decimal point together"); - errors.add("multiple decimal points"); - errors.add("cannot use \"S\" twice"); - errors.add("must be ahead of \"PR\""); - errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); - errors.add("cannot use \"S\" and \"SG\" together"); - errors.add("cannot use \"S\" and \"MI\" together"); - errors.add("cannot use \"S\" and \"PL\" together"); - errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); - errors.add("is not a number"); - } - - private static void addBitStringOperationErrors(ExpectedErrors errors) { - errors.add("cannot XOR bit strings of different sizes"); - errors.add("cannot AND bit strings of different sizes"); - errors.add("cannot OR bit strings of different sizes"); - errors.add("must be type boolean, not type text"); - } - - private static void addFunctionErrors(ExpectedErrors errors) { - errors.add("out of valid range"); // get_bit/get_byte - errors.add("cannot take logarithm of a negative number"); - errors.add("cannot take logarithm of zero"); - errors.add("requested character too large for encoding"); // chr - errors.add("null character not permitted"); // chr - errors.add("requested character not valid for encoding"); // chr - errors.add("requested length too large"); // repeat - errors.add("invalid memory alloc request size"); // repeat - errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii - errors.add("negative substring length not allowed"); // substr - errors.add("invalid mask length"); // set_masklen - } - - private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { - errors.add("is not a valid hexadecimal digit"); - } - - public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { - errors.add("range lower bound must be less than or equal to range upper bound"); - errors.add("result of range difference would not be contiguous"); - errors.add("out of range"); - errors.add("malformed range literal"); - errors.add("result of range union would not be contiguous"); - } - - public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { - errors.add("value too long for type character"); - errors.add("not found in view targetlist"); - } - public static boolean appendDataType(YSQLDataType type, StringBuilder sb, boolean allowSerial, boolean generateOnlyKnown, List opClasses) throws AssertionError { boolean serial = false; @@ -310,7 +190,7 @@ private static void addTableConstraint(StringBuilder sb, YSQLTable table, YSQLGl TableConstraints t, ExpectedErrors errors) { List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); List otherColumns; - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); switch (t) { case CHECK: sb.append("CHECK("); @@ -381,27 +261,6 @@ private static void deleteOrUpdateAction(StringBuilder sb) { sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); } - public static void addGroupingErrors(ExpectedErrors errors) { - errors.add("non-integer constant in GROUP BY"); // TODO - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("is not in select list"); - errors.add("aggregate functions are not allowed in GROUP BY"); - } - - public static void addViewErrors(ExpectedErrors errors) { - errors.add("already exists"); - errors.add("cannot drop columns from view"); - errors.add("non-integer constant in ORDER BY"); // TODO - errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO - errors.add("cannot change data type of view column"); - errors.add("specified more than once"); // TODO - errors.add("materialized views must not use temporary tables or views"); - errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); - errors.add("is not a view"); - errors.add("non-integer constant in DISTINCT ON"); - errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); - } - public enum TableConstraints { CHECK, UNIQUE, PRIMARY_KEY, FOREIGN_KEY } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java index e2eeb1ceb..e0128707b 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -3,6 +3,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; @@ -35,7 +36,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { sb.append( YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, table.getColumns()))); } - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); errors.add("out of range"); errors.add("cannot cast"); errors.add("invalid input syntax for"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index 9f814025d..6077dcb1e 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -8,6 +8,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -120,7 +121,7 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { errors.add("functions in index expression must be marked IMMUTABLE"); errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java index 1817891b3..c26459f31 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLInsertGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; @@ -22,9 +23,9 @@ public static SQLQueryAdapter insert(YSQLGlobalState globalState) { YSQLTable table = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); ExpectedErrors errors = new ExpectedErrors(); errors.add("cannot insert into column"); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonInsertUpdateErrors(errors); - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonInsertUpdateErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); errors.add("multiple assignments to same column"); errors.add("violates foreign key constraint"); errors.add("value too long for type character varying"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java index 5e376ab4c..37945c7ac 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -57,8 +58,8 @@ public YSQLTableGenerator(String tableName, boolean generateOnlyKnown, YSQLGloba errors.add("Cannot split table that does not have primary key"); errors.add("inherits from generated column but specifies identity"); errors.add("inherits from generated column but specifies default"); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonTableErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonTableErrors(errors); } public static SQLQueryAdapter generate(String tableName, boolean generateOnlyKnown, YSQLGlobalState globalState) { @@ -148,7 +149,7 @@ private void generatePartitionBy() { errors.add("unsupported UNIQUE constraint with partition key definition"); errors.add("does not accept data type"); int n = partitionOption.contentEquals("LIST") ? 1 : Randomly.smallNumber() + 1; - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); for (int i = 0; i < n; i++) { if (i != 0) { sb.append(", "); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index fe8e71ada..06e762440 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -5,6 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -33,7 +34,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { // table multiple times errors.add("new row violates check option for view"); List columns = randomTable.getRandomNonEmptyColumnSubset(); - YSQLCommon.addCommonInsertUpdateErrors(errors); + YSQLErrors.addCommonInsertUpdateErrors(errors); for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -62,7 +63,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { errors.add("violates check constraint"); errors.add("could not determine which collation to use for string comparison"); errors.add("but expression is of type"); - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { sb.append(" WHERE "); YSQLExpression where = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java index a34be987b..f70891cc2 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -4,6 +4,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLVisitor; import sqlancer.yugabyte.ysql.ast.YSQLSelect; @@ -50,9 +51,9 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { YSQLSelect select = YSQLRandomQueryGenerator.createRandomQuery(nrColumns, globalState); sb.append(YSQLVisitor.asString(select)); sb.append(")"); - YSQLCommon.addGroupingErrors(errors); - YSQLCommon.addViewErrors(errors); - YSQLCommon.addCommonExpressionErrors(errors); + YSQLErrors.addGroupingErrors(errors); + YSQLErrors.addViewErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, true); } diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java index 1eb6b9ff4..d6763e799 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java @@ -13,9 +13,9 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; public class YSQLCatalog implements TestOracle { @@ -40,8 +40,8 @@ public YSQLCatalog(YSQLGlobalState globalState) { this.con = state.getConnection(); this.logger = state.getLogger(); this.options = state.getOptions(); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonFetchErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); } private YSQLProvider.Action getRandomAction(List actions) { diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java index a5af7111f..0aa16993e 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java @@ -7,10 +7,10 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.YSQLVisitor; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLRandomQueryGenerator; public class YSQLFuzzer implements TestOracle { @@ -21,10 +21,10 @@ public class YSQLFuzzer implements TestOracle { public YSQLFuzzer(YSQLGlobalState globalState) { this.globalState = globalState; - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonFetchErrors(errors); - YSQLCommon.addGroupingErrors(errors); - YSQLCommon.addViewErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); + YSQLErrors.addGroupingErrors(errors); + YSQLErrors.addViewErrors(errors); // remove timeout error from scope errors.add("canceling statement due to statement timeout"); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java index 863b63398..6b2a57560 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -16,6 +16,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; @@ -29,7 +30,6 @@ import sqlancer.yugabyte.ysql.ast.YSQLJoin; import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPBase; @@ -40,8 +40,8 @@ public class YSQLNoRECOracle extends NoRECBase implements TestO public YSQLNoRECOracle(YSQLGlobalState globalState) { super(globalState); this.s = globalState.getSchema(); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonFetchErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); } public static List getJoinStatements(YSQLGlobalState globalState, List columns, diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java index 95ac268f4..6099f5b87 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java @@ -10,6 +10,7 @@ import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; @@ -21,7 +22,6 @@ import sqlancer.yugabyte.ysql.ast.YSQLExpression; import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; public class YSQLPivotedQuerySynthesisOracle @@ -31,8 +31,8 @@ public class YSQLPivotedQuerySynthesisOracle public YSQLPivotedQuerySynthesisOracle(YSQLGlobalState globalState) throws SQLException { super(globalState); - YSQLCommon.addCommonExpressionErrors(errors); - YSQLCommon.addCommonFetchErrors(errors); + YSQLErrors.addCommonExpressionErrors(errors); + YSQLErrors.addCommonFetchErrors(errors); } /* diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java index aa6dfd530..26e4d9179 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -14,6 +14,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLVisitor; @@ -27,7 +28,6 @@ import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation.PrefixOperator; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { @@ -38,7 +38,7 @@ public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { public YSQLTLPAggregateOracle(YSQLGlobalState state) { super(state); - YSQLCommon.addGroupingErrors(errors); + YSQLErrors.addGroupingErrors(errors); } @Override diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java index 359f12d92..e574391de 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -10,6 +10,7 @@ import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; @@ -21,7 +22,6 @@ import sqlancer.yugabyte.ysql.ast.YSQLExpression; import sqlancer.yugabyte.ysql.ast.YSQLJoin; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLCommon; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; @@ -35,8 +35,8 @@ public class YSQLTLPBase extends TernaryLogicPartitioningOracleBase Date: Wed, 5 Oct 2022 16:19:27 +0400 Subject: [PATCH 194/856] Tuned YCQL test --- test/sqlancer/dbms/TestYugabyte.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/sqlancer/dbms/TestYugabyte.java b/test/sqlancer/dbms/TestYugabyte.java index 083eb2c91..168982c22 100644 --- a/test/sqlancer/dbms/TestYugabyte.java +++ b/test/sqlancer/dbms/TestYugabyte.java @@ -14,7 +14,8 @@ public class TestYugabyte { @Test public void testYCQL() { assumeTrue(yugabyteIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", + assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "cassandra", "--password", "cassandra", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "ycql")); } From f58ee48dfbf5bf2e43a8b82ec16fbab35edfa29c Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 16:20:37 +0400 Subject: [PATCH 195/856] Reformat YCQL test --- test/sqlancer/dbms/TestYugabyte.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test/sqlancer/dbms/TestYugabyte.java b/test/sqlancer/dbms/TestYugabyte.java index 168982c22..57798924d 100644 --- a/test/sqlancer/dbms/TestYugabyte.java +++ b/test/sqlancer/dbms/TestYugabyte.java @@ -14,9 +14,10 @@ public class TestYugabyte { @Test public void testYCQL() { assumeTrue(yugabyteIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "cassandra", "--password", "cassandra", "--num-threads", - "4", "--num-queries", TestConfig.NUM_QUERIES, "ycql")); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "cassandra", "--password", "cassandra", "--num-threads", "4", "--num-queries", + TestConfig.NUM_QUERIES, "ycql")); } @Test From cae924702c141eca43fe5efb8a2afcebb51afef6 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 16:48:43 +0400 Subject: [PATCH 196/856] Fixed YugabyteDB naming in CI --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7bf958bc1..68ef0b2bf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -298,7 +298,7 @@ jobs: run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test yugabyte: - name: DBMS Tests (Yugabyte) + name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 From cfa76ecb51f94509cf2d75532197b98488bf6ead Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 17:10:46 +0400 Subject: [PATCH 197/856] Reduced number of threads in test to avoid sporadic failures Added some allowed errors and DDL locks for YCQL --- src/sqlancer/yugabyte/ycql/YCQLProvider.java | 31 ++++++++++--------- .../ysql/gen/YSQLTableGroupGenerator.java | 1 + test/sqlancer/dbms/TestYugabyte.java | 4 +-- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/sqlancer/yugabyte/ycql/YCQLProvider.java b/src/sqlancer/yugabyte/ycql/YCQLProvider.java index 59624f588..b82f5b0a2 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLProvider.java +++ b/src/sqlancer/yugabyte/ycql/YCQLProvider.java @@ -1,6 +1,7 @@ package sqlancer.yugabyte.ycql; import static sqlancer.yugabyte.ycql.YCQLSchema.getTableNames; +import static sqlancer.yugabyte.ysql.YSQLProvider.DDL_LOCK; import java.sql.Connection; import java.sql.DriverManager; @@ -137,22 +138,24 @@ public SQLConnection createDatabase(YCQLGlobalState globalState) throws SQLExcep final Connection connection = DriverManager.getConnection( String.format(url, host, port, "system_schema", globalState.getDbmsSpecificOptions().datacenter)); - try (Statement stmt = connection.createStatement()) { - try { - stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); - } catch (Exception se) { - // try again - List tableNames = getTableNames( - new SQLConnection(DriverManager.getConnection(String.format(url, host, port, - globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))), - globalState.getDatabaseName()); - for (String tableName : tableNames) { - stmt.execute("DROP TABLE " + globalState.getDatabaseName() + "." + tableName); + synchronized (DDL_LOCK) { + try (Statement stmt = connection.createStatement()) { + try { + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); + } catch (Exception se) { + // try again + List tableNames = getTableNames( + new SQLConnection(DriverManager.getConnection(String.format(url, host, port, + globalState.getDatabaseName(), globalState.getDbmsSpecificOptions().datacenter))), + globalState.getDatabaseName()); + for (String tableName : tableNames) { + stmt.execute("DROP TABLE " + globalState.getDatabaseName() + "." + tableName); + } + stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); } - stmt.execute("DROP KEYSPACE IF EXISTS " + globalState.getDatabaseName()); - } - stmt.execute("CREATE KEYSPACE IF NOT EXISTS " + globalState.getDatabaseName()); + stmt.execute("CREATE KEYSPACE IF NOT EXISTS " + globalState.getDatabaseName()); + } } return new SQLConnection(DriverManager.getConnection(String.format(url, host, port, diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java index da1b5c573..8bc292348 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLTableGroupGenerator.java @@ -19,6 +19,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { StringBuilder sb = new StringBuilder("CREATE TABLEGROUP "); String tableGroupName = "tg" + UNIQUE_TABLEGROUP_COUNTER.incrementAndGet(); sb.append(tableGroupName); + errors.add("cannot use tablegroups in a colocated database"); return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/test/sqlancer/dbms/TestYugabyte.java b/test/sqlancer/dbms/TestYugabyte.java index 57798924d..10387948b 100644 --- a/test/sqlancer/dbms/TestYugabyte.java +++ b/test/sqlancer/dbms/TestYugabyte.java @@ -16,7 +16,7 @@ public void testYCQL() { assumeTrue(yugabyteIsAvailable); assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "cassandra", "--password", "cassandra", "--num-threads", "4", "--num-queries", + "cassandra", "--password", "cassandra", "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "ycql")); } @@ -25,7 +25,7 @@ public void testYSQL() { assumeTrue(yugabyteIsAvailable); assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "yugabyte", "--password", "yugabyte", "--num-threads", "4", "--num-queries", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "ysql")); } } From 7d46ee0c7a2173825ad3ca6ac60ee84069e26754 Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 18:07:31 +0400 Subject: [PATCH 198/856] Added bug14330 --- src/sqlancer/yugabyte/YugabyteBugs.java | 3 +++ .../yugabyte/ycql/gen/YCQLExpressionGenerator.java | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/src/sqlancer/yugabyte/YugabyteBugs.java b/src/sqlancer/yugabyte/YugabyteBugs.java index 424ba3217..713ae6f88 100644 --- a/src/sqlancer/yugabyte/YugabyteBugs.java +++ b/src/sqlancer/yugabyte/YugabyteBugs.java @@ -5,6 +5,9 @@ public final class YugabyteBugs { // https://github.com/yugabyte/yugabyte-db/issues/11357 public static boolean bug11357 = true; + // https://github.com/yugabyte/yugabyte-db/issues/14330 + public static boolean bug14330 = true; + private YugabyteBugs() { } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java index 296be911d..77a4076cf 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.ColumnReferenceNode; @@ -23,6 +24,8 @@ import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import static sqlancer.yugabyte.YugabyteBugs.bug14330; + public final class YCQLExpressionGenerator extends UntypedExpressionGenerator, YCQLColumn> { private final YCQLGlobalState globalState; @@ -82,6 +85,10 @@ protected Node generateColumn() { @Override public Node generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { + if (bug14330) { + throw new IgnoreMeException(); + } + return YCQLConstant.createNullConstant(); } YCQLDataType type = YCQLDataType.getRandom(); From c3baf28af1f7689533085029481a5d4a5da5ad5e Mon Sep 17 00:00:00 2001 From: Dmitry Sherstobitov Date: Wed, 5 Oct 2022 21:40:03 +0400 Subject: [PATCH 199/856] Fixed code style for bug14330 --- src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java index 77a4076cf..7a0a7ed1b 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java @@ -1,5 +1,7 @@ package sqlancer.yugabyte.ycql.gen; +import static sqlancer.yugabyte.YugabyteBugs.bug14330; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -24,8 +26,6 @@ import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; -import static sqlancer.yugabyte.YugabyteBugs.bug14330; - public final class YCQLExpressionGenerator extends UntypedExpressionGenerator, YCQLColumn> { private final YCQLGlobalState globalState; From ffd1ac9a22671975ca737146d1da5dc9a5e4d4f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 08:20:41 +0000 Subject: [PATCH 200/856] Bump spotbugs-maven-plugin from 4.7.1.1 to 4.7.2.0 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.1.1 to 4.7.2.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.1.1...spotbugs-maven-plugin-4.7.2.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fdb287a10..1b550a2bc 100644 --- a/pom.xml +++ b/pom.xml @@ -252,7 +252,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.1.1 + 4.7.2.0 spotbugs From dd27b0b69887ad9eb681b062eb21959975b0d07c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 08:46:24 +0000 Subject: [PATCH 201/856] Bump junit-jupiter-engine from 5.9.0 to 5.9.1 Bumps [junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.9.0 to 5.9.1. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.9.0...r5.9.1) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1b550a2bc..1b76c1dd7 100644 --- a/pom.xml +++ b/pom.xml @@ -319,7 +319,7 @@ org.junit.jupiter junit-jupiter-engine - 5.9.0 + 5.9.1 test From fd39eea8b00f46c4d930f3814ffb0204496b8792 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 09:42:26 +0000 Subject: [PATCH 202/856] Bump mariadb-java-client from 3.0.7 to 3.0.8 Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.7 to 3.0.8. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/compare/3.0.7...3.0.8) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1b76c1dd7..529f86a51 100644 --- a/pom.xml +++ b/pom.xml @@ -309,7 +309,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.7 + 3.0.8 org.duckdb From b429f5218ea8470a539659354c6b2d8a994db1de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 6 Oct 2022 10:14:01 +0000 Subject: [PATCH 203/856] Bump duckdb_jdbc from 0.4.0 to 0.5.1 Bumps [duckdb_jdbc](https://github.com/duckdb/duckdb) from 0.4.0 to 0.5.1. - [Release notes](https://github.com/duckdb/duckdb/releases) - [Commits](https://github.com/duckdb/duckdb/compare/v0.4.0...v0.5.1) --- updated-dependencies: - dependency-name: org.duckdb:duckdb_jdbc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 529f86a51..bca0450bc 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 0.4.0 + 0.5.1 org.junit.jupiter From 6850356f27a125ca777477c5476f7ec214da1f18 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Fri, 7 Oct 2022 10:12:44 +0800 Subject: [PATCH 204/856] Fix ALTER grammar in TiDB --- src/sqlancer/tidb/TiDBExpressionGenerator.java | 6 ------ src/sqlancer/tidb/TiDBSchema.java | 6 ++++++ src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java | 12 +++++------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index 20f48f04a..535647c53 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -85,9 +85,6 @@ protected TiDBExpression generateExpression(int depth) { case REGEX: return new TiDBRegexOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBRegexOperator.getRandom()); - // case COLLATE: - // return new TiDBCollate(generateExpression(depth + 1), - // Randomly.fromOptions("utf8mb4_bin", "latin1_bin", "binary", "ascii_bin", "utf8_bin")); case FUNCTION: TiDBFunction func = TiDBFunction.getRandom(); return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); @@ -100,9 +97,6 @@ protected TiDBExpression generateExpression(int depth) { } return new TiDBBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryLogicalOperator.getRandom()); - // case BINARY_ARITHMETIC: - // return new TiDBBinaryArithmeticOperation(generateExpression(depth + 1), generateExpression(depth + 1), - // TiDBBinaryArithmeticOperator.getRandom()); case CAST: return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions("BINARY", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 7caff2974..676475d4a 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -227,14 +227,20 @@ private static TiDBCompositeDataType getColumnType(String typeString) { primitiveType = TiDBDataType.TEXT; break; case "float": + size = 4; + primitiveType = TiDBDataType.FLOATING; + break; case "double": + size = 8; primitiveType = TiDBDataType.FLOATING; break; case "tinyint(1)": primitiveType = TiDBDataType.BOOL; + size = 1; break; case "null": primitiveType = TiDBDataType.INT; + size = 1; break; case "tinyint(3)": case "tinyint(4)": diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 057bb2789..a327dde39 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -9,6 +9,7 @@ import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; import sqlancer.tidb.TiDBSchema.TiDBDataType; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -25,11 +26,9 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add( "Information schema is changed during the execution of the statement(for example, table definition may be updated by other DDL ran in parallel)"); - errors.add("Data truncated"); - errors.add("Data truncation"); + errors.add("Data truncat"); errors.add("without a key length"); - errors.add("charset"); - errors.add("not supported"); + errors.add("supported"); errors.add("SQL syntax"); errors.add("can't drop"); StringBuilder sb = new StringBuilder("ALTER TABLE "); @@ -46,8 +45,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append("MODIFY "); sb.append(column.getName()); sb.append(" "); - sb.append(TiDBDataType.getRandom()); - errors.add("Unsupported modify column"); + sb.append(TiDBCompositeDataType.getRandom().toString()); break; case DROP_COLUMN: sb.append(" DROP "); @@ -102,7 +100,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append(" "); sb.append(column.getName()); sb.append(" "); - sb.append(column.getType().getPrimitiveDataType()); + sb.append(column.getType().toString()); sb.append(" NOT NULL "); errors.add("Invalid use of NULL value"); errors.add("Unsupported modify column:"); From f35e92bbf63eb3153589e66f71239d97441a8891 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 03:56:45 +0000 Subject: [PATCH 205/856] build(deps): bump maven-dependency-plugin from 3.2.0 to 3.3.0 Bumps [maven-dependency-plugin](https://github.com/apache/maven-dependency-plugin) from 3.2.0 to 3.3.0. - [Release notes](https://github.com/apache/maven-dependency-plugin/releases) - [Commits](https://github.com/apache/maven-dependency-plugin/compare/maven-dependency-plugin-3.2.0...maven-dependency-plugin-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-dependency-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index bca0450bc..dc01ce457 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.2.0 + 3.3.0 copy-dependencies From 12059f0a30d22754f42d1f8be77deb66b535c298 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 04:10:46 +0000 Subject: [PATCH 206/856] Bump maven-jar-plugin from 3.2.2 to 3.3.0 Bumps [maven-jar-plugin](https://github.com/apache/maven-jar-plugin) from 3.2.2 to 3.3.0. - [Release notes](https://github.com/apache/maven-jar-plugin/releases) - [Commits](https://github.com/apache/maven-jar-plugin/compare/maven-jar-plugin-3.2.2...maven-jar-plugin-3.3.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-jar-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dc01ce457..d446aeb67 100644 --- a/pom.xml +++ b/pom.xml @@ -175,7 +175,7 @@ org.apache.maven.plugins maven-jar-plugin - 3.2.2 + 3.3.0 true From 903272af7e128bf6ae134495ad494207dc304faa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 7 Oct 2022 05:39:35 +0000 Subject: [PATCH 207/856] Bump maven-shade-plugin from 3.3.0 to 3.4.0 Bumps [maven-shade-plugin](https://github.com/apache/maven-shade-plugin) from 3.3.0 to 3.4.0. - [Release notes](https://github.com/apache/maven-shade-plugin/releases) - [Commits](https://github.com/apache/maven-shade-plugin/compare/maven-shade-plugin-3.3.0...maven-shade-plugin-3.4.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-shade-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d446aeb67..d5d73cdd0 100644 --- a/pom.xml +++ b/pom.xml @@ -44,7 +44,7 @@ org.apache.maven.plugins maven-shade-plugin - 3.3.0 + 3.4.0 package From 4a1977fd0a75215da788a74ada95cbb153b9aa0b Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 9 Oct 2022 14:55:41 +0800 Subject: [PATCH 208/856] docs: add databend info Signed-off-by: Yisong Han --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d163f049c..eb0fcd779 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | | OceanBase | Working | Untyped | | | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | - +| Databend | Working | Typed | | # Using SQLancer From b3096a110c0997dfd4d58a0e3a6b1f699c2dc8d4 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 9 Oct 2022 15:41:20 +0800 Subject: [PATCH 209/856] chore: update unit test and databenderror etc. Signed-off-by: Yisong Han --- .gitignore | 1 + src/sqlancer/databend/DatabendErrors.java | 110 +++------------------- test/sqlancer/dbms/TestDatabend.java | 1 - 3 files changed, 14 insertions(+), 98 deletions(-) diff --git a/.gitignore b/.gitignore index 48efcf83a..a3ecf6076 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ target/ .settings/org.eclipse.core.resources.prefs .settings/org.eclipse.m2e.core.prefs .settings/org.eclipse.jdt.core.prefs +.vscode .project .checkstyle *.DS_Store diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 60ea7ca9c..7e1f80c4e 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -8,109 +8,25 @@ private DatabendErrors() { } public static void addExpressionErrors(ExpectedErrors errors) { - errors.add("with non-constant precision is not supported"); - errors.add("Like pattern must not end with escape character"); - errors.add("Could not convert string"); - errors.add("ORDER term out of range - should be between "); - errors.add("You might need to add explicit type casts."); - errors.add("can't be cast because the value is out of range for the destination type"); - errors.add("field value out of range"); - errors.add("Unimplemented type for cast"); - - errors.add("Escape string must be empty or one character."); - errors.add("Type mismatch when combining rows"); // BETWEEN - - errors.add("invalid UTF-8"); // TODO - errors.add("String value is not valid UTF8"); - - errors.add("Invalid TypeId "); // TODO - - errors.add("GROUP BY clause cannot contain aggregates!"); // investigate - - addRegexErrors(errors); - - addFunctionErrors(errors); - - errors.add("Overflow in multiplication"); - errors.add("Out of Range"); - errors.add("Date out of range"); - - // collate - errors.add("Cannot combine types with different collation!"); - errors.add("collations are only supported for type varchar"); - - // // https://github.com/cwida/Databend/issues/532 - errors.add("Not implemented type: DATE"); - errors.add("Not implemented type: TIMESTAMP"); - errors.add("Like pattern must not end with escape character!"); // LIKE - - errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a - // table supports rowids - - errors.add("does not have a column named"); // TODO: this only happens for views whose underlying table has a - // removed column - errors.add("Contents of view were altered: types don't match!"); - errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); - // TODO Databend待修复的bug（union schema error mismatch） - errors.add("unexpected end of file (failed to fill whole buffer)"); - } - - private static void addRegexErrors(ExpectedErrors errors) { - errors.add("missing ]"); - errors.add("missing )"); - errors.add("invalid escape sequence"); - errors.add("no argument for repetition operator: "); - errors.add("bad repetition operator"); - errors.add("trailing \\"); - errors.add("invalid perl operator"); - errors.add("invalid character class range"); - errors.add("width is not integer"); - } - - private static void addFunctionErrors(ExpectedErrors errors) { - errors.add("SUBSTRING cannot handle negative lengths"); - errors.add("is undefined outside [-1,1]"); // ACOS etc - errors.add("invalid type specifier"); // PRINTF - errors.add("argument index out of range"); // PRINTF - errors.add("invalid format string"); // PRINTF - errors.add("number is too big"); // PRINTF - errors.add("Like pattern must not end with escape character!"); // LIKE - errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part - errors.add("extract specifier"); // date_part - errors.add("not recognized"); // date_part - errors.add("not supported"); // date_part - errors.add("Failed to cast"); - errors.add("Conversion Error"); - errors.add("Could not cast value"); - errors.add("Insufficient padding in RPAD"); // RPAD - errors.add("Could not choose a best candidate function for the function call"); // monthname - errors.add("expected a numeric precision field"); // ROUND - errors.add("with non-constant precision is not supported"); // ROUND + errors.add("Division by zero"); + + /* + * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE + * databend2; CREATE TABLE t0(c0VARCHAR VARCHAR NULL, c1VARCHAR VARCHAR NULL, c2FLOAT FLOAT NOT NULL + * DEFAULT(NULL)); CREATE TABLE t1(c0INT BIGINT NULL); INSERT INTO t0(c1varchar, c0varchar) VALUES ('067596', + * '19'), ('', '87'); + */ + errors.add("Can't cast column from null into non-nullable type"); } public static void addInsertErrors(ExpectedErrors errors) { - addRegexErrors(errors); - addFunctionErrors(errors); - - errors.add("NOT NULL constraint failed"); - errors.add("PRIMARY KEY or UNIQUE constraint violated"); - errors.add("duplicate key"); - errors.add("can't be cast because the value is out of range for the destination type"); - errors.add("Could not convert string"); - errors.add("Unimplemented type for cast"); - errors.add("field value out of range"); - errors.add("CHECK constraint failed"); - errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid - errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if - // the table has a primary key - errors.add("Could not cast value"); - errors.add("create unique index, table contains duplicate data"); - errors.add("Failed to cast"); + errors.add("Division by zero"); + errors.add("Can't cast column from null into non-nullable type"); } public static void addGroupByErrors(ExpectedErrors errors) { - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("GROUP BY term out of range"); + errors.add("Division by zero"); + errors.add("Can't cast column from null into non-nullable type"); } } diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index a9575fd7d..326a43283 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -21,7 +21,6 @@ public void testDatabendNoREC() { "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); } - // TODO Databend待修复的bug（union schema error mismatch）https://github.com/datafuselabs/databend/issues/7463 @Test public void testDatabendTLPQueryPartitioning() { String databendAvailable = System.getenv("DATABEND_AVAILABLE"); From 72434c08523b5bb92a334b955e6c0aeb02856311 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 06:23:48 +0000 Subject: [PATCH 210/856] build(deps): bump checkstyle from 10.3.3 to 10.3.4 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3.3 to 10.3.4. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3.3...checkstyle-10.3.4) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d5d73cdd0..4c4f5b61c 100644 --- a/pom.xml +++ b/pom.xml @@ -209,7 +209,7 @@ com.puppycrawl.tools checkstyle - 10.3.3 + 10.3.4 From 93c26a4b52ad56cc80358a877a9ebcb0c8a9aa06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Oct 2022 09:15:44 +0000 Subject: [PATCH 211/856] build(deps): bump spotbugs-maven-plugin from 4.7.2.0 to 4.7.2.1 Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.2.0 to 4.7.2.1. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.2.0...spotbugs-maven-plugin-4.7.2.1) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 4c4f5b61c..1f9ebbe36 100644 --- a/pom.xml +++ b/pom.xml @@ -252,7 +252,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.2.0 + 4.7.2.1 spotbugs From 7b372ede548727584d69ee5855cb832cce88a887 Mon Sep 17 00:00:00 2001 From: Suri Zhang Date: Wed, 12 Oct 2022 13:56:27 +0800 Subject: [PATCH 212/856] Add Provider and Schema for QuestDB --- pom.xml | 7 +- src/sqlancer/GlobalState.java | 2 +- src/sqlancer/questdb/QuestDBErrors.java | 4 + src/sqlancer/questdb/QuestDBOptions.java | 55 ++++ src/sqlancer/questdb/QuestDBProvider.java | 106 ++++++ src/sqlancer/questdb/QuestDBSchema.java | 308 ++++++++++++++++++ .../questdb/gen/QuestDBDeleteGenerator.java | 15 + .../questdb/gen/QuestDBIndexGenerator.java | 14 + .../questdb/gen/QuestDBInsertGenerator.java | 35 ++ .../questdb/gen/QuestDBTableGenerator.java | 43 +++ .../questdb/gen/QuestDBUpdateGenerator.java | 16 + .../questdb/gen/QuestDBViewGenerator.java | 14 + 12 files changed, 617 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/questdb/QuestDBErrors.java create mode 100644 src/sqlancer/questdb/QuestDBOptions.java create mode 100644 src/sqlancer/questdb/QuestDBProvider.java create mode 100644 src/sqlancer/questdb/QuestDBSchema.java create mode 100644 src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBIndexGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBInsertGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBTableGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBViewGenerator.java diff --git a/pom.xml b/pom.xml index fdb287a10..9a736cc77 100644 --- a/pom.xml +++ b/pom.xml @@ -347,6 +347,11 @@ arangodb-java-driver 6.9.0 + + org.questdb + questdb + 6.5.3 + @@ -446,4 +451,4 @@ - + \ No newline at end of file diff --git a/src/sqlancer/GlobalState.java b/src/sqlancer/GlobalState.java index 64b5c731e..2b93012c2 100644 --- a/src/sqlancer/GlobalState.java +++ b/src/sqlancer/GlobalState.java @@ -131,7 +131,7 @@ public S getSchema() { try { updateSchema(); } catch (Exception e) { - throw new AssertionError(); + throw new AssertionError(e.getMessage()); } } return schema; diff --git a/src/sqlancer/questdb/QuestDBErrors.java b/src/sqlancer/questdb/QuestDBErrors.java new file mode 100644 index 000000000..534c5c41d --- /dev/null +++ b/src/sqlancer/questdb/QuestDBErrors.java @@ -0,0 +1,4 @@ +package sqlancer.questdb; + +public class QuestDBErrors { +} diff --git a/src/sqlancer/questdb/QuestDBOptions.java b/src/sqlancer/questdb/QuestDBOptions.java new file mode 100644 index 000000000..ad2b593af --- /dev/null +++ b/src/sqlancer/questdb/QuestDBOptions.java @@ -0,0 +1,55 @@ +package sqlancer.questdb; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.questdb.QuestDBOptions.QuestDBOracleFactory; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +@Parameters(separators = "=", commandDescription = "QuestDB (default port: " + QuestDBOptions.DEFAULT_PORT + + " default host: " + QuestDBOptions.DEFAULT_HOST + ")") +public class QuestDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 8812; + + @Parameter(names = "--oracle") + public QuestDBOracleFactory oracle = QuestDBOracleFactory.WHERE; + + public enum QuestDBOracleFactory implements OracleFactory { + // TODO: implement test oracles + WHERE { + @Override + public TestOracle create(QuestDBGlobalState globalState) throws SQLException { + return null; + } + }; + + } + + @Parameter(names = "--username", description = "The user name used to log into QuestDB") + private String userName = "admin"; // NOPMD + + @Parameter(names = "--password", description = "The password used to log into QuestDB") + private String password = "quest"; // NOPMD + + @Override + public List getTestOracleFactory() { + return Arrays.asList(oracle); + } + + public String getUserName() { + return userName; + } + + public String getPassword() { + return password; + } + +} diff --git a/src/sqlancer/questdb/QuestDBProvider.java b/src/sqlancer/questdb/QuestDBProvider.java new file mode 100644 index 000000000..997ab40da --- /dev/null +++ b/src/sqlancer/questdb/QuestDBProvider.java @@ -0,0 +1,106 @@ +package sqlancer.questdb; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.gen.QuestDBDeleteGenerator; +import sqlancer.questdb.gen.QuestDBIndexGenerator; +import sqlancer.questdb.gen.QuestDBInsertGenerator; +import sqlancer.questdb.gen.QuestDBTableGenerator; + +@AutoService(DatabaseProvider.class) +public class QuestDBProvider extends SQLProviderAdapter { + public QuestDBProvider() { + super(QuestDBGlobalState.class, QuestDBOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(QuestDBInsertGenerator::getQuery), // + CREATE_INDEX(QuestDBIndexGenerator::getQuery), // + DELETE(QuestDBDeleteGenerator::generate); // + // TODO: maybe implement these later + // UPDATE(QuestDBUpdateGenerator::getQuery), // + // CREATE_VIEW(QuestDBViewGenerator::generate), // + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(QuestDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + public static class QuestDBGlobalState extends SQLGlobalState { + + @Override + protected QuestDBSchema readSchema() throws SQLException { + return QuestDBSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(QuestDBGlobalState globalState) throws Exception { + // TODO: should follow duckdb or cockrachdb? what's the difference between generate and create db? + } + + @Override + public SQLConnection createDatabase(QuestDBGlobalState globalState) throws Exception { + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = QuestDBOptions.DEFAULT_HOST; + } + if (port == sqlancer.MainOptions.NO_SET_PORT) { + port = QuestDBOptions.DEFAULT_PORT; + } + // TODO(anxing): maybe not hardcode here... + String databaseName = "qdb"; + String tableName = "test"; + String url = String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName); + // use QuestDB default username & password for Postgres JDBC + Properties properties = new Properties(); + properties.setProperty("user", globalState.getDbmsSpecificOptions().getUserName()); + properties.setProperty("password", globalState.getDbmsSpecificOptions().getPassword()); + properties.setProperty("sslmode", "disable"); + + Connection con = DriverManager.getConnection(url, properties); + // QuestDB cannot create or drop `DATABASE`, can only create or drop `TABLE` + globalState.getState().logStatement("DROP TABLE IF EXISTS " + tableName + " CASCADE"); + SQLQueryAdapter createTableCommand = new QuestDBTableGenerator().getQuery(globalState); + globalState.getState().logStatement(createTableCommand); + + try (Statement s = con.createStatement()) { + s.execute("DROP TABLE IF EXISTS " + tableName); + } + try (Statement s = con.createStatement()) { + s.execute(createTableCommand.getQueryString()); + } + con.close(); + con = DriverManager.getConnection(url, properties); + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "questdb"; + } + +} diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java new file mode 100644 index 000000000..ef2026706 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -0,0 +1,308 @@ +package sqlancer.questdb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; + +public class QuestDBSchema extends AbstractSchema { + + public enum QuestDBDataType { + + BOOLEAN, CHAR, + /* STRING, */ + INT, FLOAT, + /* SYMBOL, */ + DATE, TIMESTAMP, + /* GEOHASH, */ + NULL; + + public static QuestDBDataType getRandomWithoutNull() { + QuestDBDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == QuestDBDataType.NULL); + return dt; + } + + } + + public static class QuestDBCompositeDataType { + + private final QuestDBDataType dataType; + + private final int size; + + private final boolean isNullable; + + public QuestDBCompositeDataType(QuestDBDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + + switch (dataType) { + case INT: + switch (size) { + case 1: + case 2: + isNullable = false; + break; + default: + isNullable = true; + break; + } + break; + case BOOLEAN: + isNullable = false; + break; + default: + isNullable = true; + } + } + + public QuestDBDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public boolean isNullable() { + return isNullable; + } + + public static QuestDBCompositeDataType getRandomWithoutNull() { + QuestDBDataType type = QuestDBDataType.getRandomWithoutNull(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8, 32); + break; + case BOOLEAN: + case CHAR: + case DATE: + case TIMESTAMP: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new QuestDBCompositeDataType(type, size); + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 1: + return Randomly.fromOptions("BYTE"); + case 2: + return Randomly.fromOptions("SHORT"); + case 4: + return Randomly.fromOptions("INT"); + default: + throw new AssertionError(size); + } + case CHAR: + return "CHAR"; + case FLOAT: + switch (size) { + case 4: + return Randomly.fromOptions("FLOAT"); + case 8: + return Randomly.fromOptions(/* "DOUBLE", */"LONG"); + case 32: + return Randomly.fromOptions("LONG256"); + default: + throw new AssertionError(size); + } + case BOOLEAN: + return Randomly.fromOptions("BOOLEAN"); + case TIMESTAMP: + return Randomly.fromOptions("TIMESTAMP"); + case DATE: + return Randomly.fromOptions("DATE"); + case NULL: + return Randomly.fromOptions("NULL"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + } + + public static class QuestDBColumn extends AbstractTableColumn { + private final boolean isIndexed; + private final boolean isNullable; + + public QuestDBColumn(String name, QuestDBCompositeDataType columnType, boolean isIndexed) { + super(name, null, columnType); + this.isIndexed = isIndexed; + this.isNullable = columnType.isNullable(); + } + + public boolean isIndexed() { + return isIndexed; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class QuestDBTables extends AbstractTables { + public static final Set RESERVED_TABLES = new HashSet<>( + Arrays.asList("sys.column_versions_purge_log", "telemetry_config", "telemetry")); + + public QuestDBTables(List tables) { + super(tables); + } + } + + public QuestDBSchema(List databaseTables) { + super(databaseTables); + } + + public QuestDBTables getRandomTableNonEmptyTables() { + return new QuestDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + private static QuestDBCompositeDataType getColumnType(String typeString) { + QuestDBDataType primitiveType; + int size = -1; + + switch (typeString) { + case "INT": + primitiveType = QuestDBDataType.INT; + size = 4; + break; + case "CHAR": + primitiveType = QuestDBDataType.CHAR; + break; + case "FLOAT": + primitiveType = QuestDBDataType.FLOAT; + size = 4; + break; + case "LONG": + primitiveType = QuestDBDataType.FLOAT; + size = 8; + break; + case "LONG256": + primitiveType = QuestDBDataType.FLOAT; + size = 32; + break; + case "BOOLEAN": + primitiveType = QuestDBDataType.BOOLEAN; + break; + case "DATE": + primitiveType = QuestDBDataType.DATE; + break; + case "TIMESTAMP": + primitiveType = QuestDBDataType.TIMESTAMP; + break; + case "BYTE": + primitiveType = QuestDBDataType.INT; + size = 1; + break; + case "SHORT": + primitiveType = QuestDBDataType.INT; + size = 2; + break; + case "NULL": + primitiveType = QuestDBDataType.NULL; + break; + default: + throw new AssertionError(typeString); + } + return new QuestDBCompositeDataType(primitiveType, size); + } + + public static class QuestDBTable extends AbstractRelationalTable { + + public QuestDBTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static QuestDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + QuestDBTable t = new QuestDBTable(tableName, databaseColumns, isView); + for (QuestDBColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new QuestDBSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES;")) { + while (rs.next()) { + String tName = rs.getString("table"); + // exclude reserved tables for testing + if (!QuestDBTables.RESERVED_TABLES.contains(tName)) { + tableNames.add(rs.getString("table")); + } + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("SHOW COLUMNS FROM %s;", tableName))) { + while (rs.next()) { + String columnName = rs.getString("column"); + String dataType = rs.getString("type"); + boolean isIndexed = rs.getString("indexed").contains("true"); + QuestDBColumn c = new QuestDBColumn(columnName, getColumnType(dataType), isIndexed); + columns.add(c); + } + } + } + if (columns.stream().noneMatch(c -> c.isIndexed())) { + // TODO: implement an option to enable/disable rowids + columns.add(new QuestDBColumn("rowid", new QuestDBCompositeDataType(QuestDBDataType.INT, 4), false)); + } + return columns; + } + +} diff --git a/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java b/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java new file mode 100644 index 000000000..8cfefd093 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java @@ -0,0 +1,15 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBDeleteGenerator { + private QuestDBDeleteGenerator() { + + } + + public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { + // TODO + return null; + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java b/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java new file mode 100644 index 000000000..b994c22c0 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java @@ -0,0 +1,14 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBIndexGenerator { + private QuestDBIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + // TODO + return null; + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java new file mode 100644 index 000000000..807917b46 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -0,0 +1,35 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; + +public class QuestDBInsertGenerator extends AbstractInsertGenerator { + + private final QuestDBGlobalState globalState; + + // uncomment later: + // private final ExpectedErrors errors = new ExpectedErrors(); + + public QuestDBInsertGenerator(QuestDBGlobalState globalState) { + this.globalState = globalState; + } + + private SQLQueryAdapter generate() { + // TODO: below is dummy implementation to pass compilation + QuestDBSchema.QuestDBTable table = globalState.getSchema().getRandomTable(); + table.getRandomNonEmptyColumnSubset(); + return null; + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + return new QuestDBInsertGenerator(globalState).generate(); + } + + @Override + protected void insertValue(QuestDBColumn tiDBColumn) { + // TODO + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java new file mode 100644 index 000000000..360c9169b --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -0,0 +1,43 @@ +package sqlancer.questdb.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; + +public class QuestDBTableGenerator { + + public SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = "test"; // globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + sb.append(columns.get(i).getType()); + } + sb.append(");"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < sqlancer.Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + QuestDBCompositeDataType columnType = QuestDBCompositeDataType.getRandomWithoutNull(); + columns.add(new QuestDBColumn(columnName, columnType, false)); + } + return columns; + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java b/src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java new file mode 100644 index 000000000..cc7fba1df --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBUpdateGenerator.java @@ -0,0 +1,16 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBUpdateGenerator { + + private QuestDBUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + // TODO + return null; + } + +} diff --git a/src/sqlancer/questdb/gen/QuestDBViewGenerator.java b/src/sqlancer/questdb/gen/QuestDBViewGenerator.java new file mode 100644 index 000000000..0ff4d232b --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBViewGenerator.java @@ -0,0 +1,14 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; + +public final class QuestDBViewGenerator { + private QuestDBViewGenerator() { + } + + public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { + // TODO + return null; + } +} From 1108bbd91b84fbbbf5136b309f8e2245616b7591 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sat, 15 Oct 2022 12:13:32 +0800 Subject: [PATCH 213/856] fix: unstable deployment databend Signed-off-by: Yisong Han --- .github/workflows/main.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68ef0b2bf..5d069d3be 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -134,10 +134,8 @@ jobs: run: | sudo apt update sudo apt install mysql-client - LASTEST_TAG=$(curl -s GET https://api.github.com/repos/datafuselabs/databend/tags\?per_page\=1 | jq -r '.[].name') - curl -LJO https://github.com/datafuselabs/databend/releases/download/${LASTEST_TAG}/databend-${LASTEST_TAG}-x86_64-unknown-linux-musl.tar.gz - mkdir ./databend && tar xzvf databend-${LASTEST_TAG}-x86_64-unknown-linux-musl.tar.gz -C ./databend - ./databend/bin/databend-query & + docker pull datafuselabs/databend-query + docker run --net=host datafuselabs/databend-query & - name: Create SQLancer user run: mysql -uroot -h127.0.0.1 -P3307 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" - name: Run Tests From 4619e20034039f0c4527d5e56618fd64fd21ce3f Mon Sep 17 00:00:00 2001 From: kokrui Date: Mon, 17 Oct 2022 19:58:36 +0800 Subject: [PATCH 214/856] Add DuckDB canonicalization rule for -0 -> 0 --- .../duckdb/test/DuckDBQueryPartitioningBase.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 08c862973..68456ef26 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -3,7 +3,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -38,9 +37,16 @@ public DuckDBQueryPartitioningBase(DuckDBGlobalState state) { } public static String canonicalizeResultValue(String value) { - // Rule: -0.0 should be canonicalized to 0.0 - if (Objects.equals(value, "-0.0")) { + if (value == null) { + return value; + } + + switch (value) { + case "-0.0": return "0.0"; + case "-0": + return "0"; + default: } return value; From b70d306a8c76ae1af9091a23d736057354660824 Mon Sep 17 00:00:00 2001 From: kokrui Date: Mon, 17 Oct 2022 15:07:58 +0800 Subject: [PATCH 215/856] Remove now-obsolete DuckDB timestamp/date errors --- src/sqlancer/duckdb/DuckDBErrors.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index cb45bb96c..590a61d47 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -39,9 +39,6 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Cannot combine types with different collation!"); errors.add("collations are only supported for type varchar"); - // // https://github.com/cwida/duckdb/issues/532 - errors.add("Not implemented type: DATE"); - errors.add("Not implemented type: TIMESTAMP"); errors.add("Like pattern must not end with escape character!"); // LIKE errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a From 27abea1aa0694089f7e51ee5c6e80a924dc66f83 Mon Sep 17 00:00:00 2001 From: kokrui Date: Mon, 17 Oct 2022 15:17:15 +0800 Subject: [PATCH 216/856] Add expected DuckDB timestamp errors --- src/sqlancer/duckdb/DuckDBErrors.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 590a61d47..555bc2e88 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -48,6 +48,10 @@ public static void addExpressionErrors(ExpectedErrors errors) { // removed column errors.add("Contents of view were altered: types don't match!"); errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + + // timestamp + errors.add("Cannot subtract infinite timestamps"); + errors.add("Timestamp difference is out of bounds"); } private static void addRegexErrors(ExpectedErrors errors) { From 6b3a422fc4a6f21f23c2444dbfc09b853dde824c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Oct 2022 13:31:53 +0000 Subject: [PATCH 217/856] build(deps): bump jdbc-yugabytedb from 42.3.0 to 42.3.5-yb-1 Bumps [jdbc-yugabytedb](https://github.com/yugabyte/pgjdbc) from 42.3.0 to 42.3.5-yb-1. - [Release notes](https://github.com/yugabyte/pgjdbc/releases) - [Changelog](https://github.com/yugabyte/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/yugabyte/pgjdbc/commits) --- updated-dependencies: - dependency-name: com.yugabyte:jdbc-yugabytedb dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index dfd0e2be7..098975cfb 100644 --- a/pom.xml +++ b/pom.xml @@ -294,7 +294,7 @@ com.yugabyte jdbc-yugabytedb - 42.3.0 + 42.3.5-yb-1 org.xerial From 294529cdf3317150069f4a5329acc886ce0beea1 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 17 Oct 2022 17:49:39 +0200 Subject: [PATCH 218/856] more errors to ignore --- src/sqlancer/clickhouse/ClickHouseErrors.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 17993c7e4..908e5afec 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -37,10 +37,13 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); errors.add("Positional argument out of bounds"); + errors.add("Illegal value (aggregate function) for positional argument in GROUP BY"); + errors.add("Invalid escape sequence at the end of LIKE pattern"); } public static void addExpressionHavingErrors(ExpectedErrors errors) { errors.add("Memory limit"); + errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 } public static void addQueryErrors(ExpectedErrors errors) { @@ -62,6 +65,7 @@ public static void addTableManipulationErrors(ExpectedErrors errors) { errors.add("Sorting key cannot contain constants"); errors.add("Sampling expression must be present in the primary key"); errors.add("Cyclic aliases: default expression and column type are incompatible"); + errors.add("Positional argument out of bounds"); } } From 9e04f20b8ac3f7e14bc67e6bf8a037372c53127b Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 17 Oct 2022 21:00:06 +0200 Subject: [PATCH 219/856] add Having errors to TLPHaving oracle (#594) --- .../clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index 2cc2410d1..fc4f498cc 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -25,6 +25,7 @@ public ClickHouseTLPHavingOracle(ClickHouseProvider.ClickHouseGlobalState state) super(state); ClickHouseErrors.addExpectedExpressionErrors(errors); ClickHouseErrors.addGroupingErrors(errors); + ClickHouseErrors.addExpressionHavingErrors(errors); } @Override From efac7d93c18f0bb154119236dc066dd115ce9dbc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:37:59 +0000 Subject: [PATCH 220/856] build(deps): bump cassandra-jdbc-wrapper from 4.6.0 to 4.7.0 Bumps [cassandra-jdbc-wrapper](https://github.com/ing-bank/cassandra-jdbc-wrapper) from 4.6.0 to 4.7.0. - [Release notes](https://github.com/ing-bank/cassandra-jdbc-wrapper/releases) - [Changelog](https://github.com/ing-bank/cassandra-jdbc-wrapper/blob/master/CHANGELOG.md) - [Commits](https://github.com/ing-bank/cassandra-jdbc-wrapper/compare/v4.6.0...v4.7.0) --- updated-dependencies: - dependency-name: com.ing.data:cassandra-jdbc-wrapper dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 098975cfb..0b104497f 100644 --- a/pom.xml +++ b/pom.xml @@ -289,7 +289,7 @@ com.ing.data cassandra-jdbc-wrapper - 4.6.0 + 4.7.0 com.yugabyte From 4e328929ea9c6c917dd0a847bbced093f350c16b Mon Sep 17 00:00:00 2001 From: Elliana May Date: Wed, 19 Oct 2022 19:55:35 +0800 Subject: [PATCH 221/856] fix: don't pass username or password to duckdb It's an inmemory database and does not support them --- src/sqlancer/duckdb/DuckDBProvider.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 3164b7d6a..2384d894b 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -143,8 +143,7 @@ public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLExc String url = "jdbc:duckdb:" + databaseFile; tryDeleteDatabase(databaseFile); - Connection conn = DriverManager.getConnection(url, globalState.getOptions().getUserName(), - globalState.getOptions().getPassword()); + Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement(); stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); stmt.close(); From 8a16b51e8753a58bb01846f9b8d997a60aba875c Mon Sep 17 00:00:00 2001 From: Elliana May Date: Wed, 19 Oct 2022 21:32:19 +0800 Subject: [PATCH 222/856] feat: add isDefaultUsername/isDefaultPassword methods to MainOptions --- src/sqlancer/MainOptions.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index ff588de5f..476dcb43f 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -5,9 +5,12 @@ import sqlancer.Randomly.StringGenerationStrategy; +import java.util.Objects; + @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { public static final int NO_SET_PORT = -1; + public static final MainOptions DEFAULT_OPTIONS = new MainOptions(); @Parameter(names = { "--help", "-h" }, description = "Lists all supported options and commands", help = true) private boolean help; // NOPMD @@ -218,6 +221,14 @@ public boolean isHelp() { return help; } + public boolean isDefaultPassword() { + return Objects.equals(password, DEFAULT_OPTIONS.password); + } + + public boolean isDefaultUsername() { + return Objects.equals(userName, DEFAULT_OPTIONS.userName); + } + public String getDatabasePrefix() { return databasePrefix; } From 6f255a596a65bb58483124094407d7c7c4b11d1b Mon Sep 17 00:00:00 2001 From: Elliana May Date: Wed, 19 Oct 2022 21:32:42 +0800 Subject: [PATCH 223/856] feat: throw an exception if someone attempts to pass creds to DuckDB --- src/sqlancer/duckdb/DuckDBProvider.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/duckdb/DuckDBProvider.java b/src/sqlancer/duckdb/DuckDBProvider.java index 2384d894b..59b26eac7 100644 --- a/src/sqlancer/duckdb/DuckDBProvider.java +++ b/src/sqlancer/duckdb/DuckDBProvider.java @@ -11,6 +11,7 @@ import sqlancer.AbstractAction; import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; @@ -143,6 +144,11 @@ public SQLConnection createDatabase(DuckDBGlobalState globalState) throws SQLExc String url = "jdbc:duckdb:" + databaseFile; tryDeleteDatabase(databaseFile); + MainOptions options = globalState.getOptions(); + if (!(options.isDefaultUsername() && options.isDefaultPassword())) { + throw new AssertionError("DuckDB doesn't support credentials (username/password)"); + } + Connection conn = DriverManager.getConnection(url); Statement stmt = conn.createStatement(); stmt.execute("PRAGMA checkpoint_threshold='1 byte';"); From 18dcbfcd925c9b633bc9a676dea5c0b4fb1d00a9 Mon Sep 17 00:00:00 2001 From: Elliana May Date: Wed, 19 Oct 2022 21:46:22 +0800 Subject: [PATCH 224/856] chore: correct import order, per checkstyle error --- src/sqlancer/MainOptions.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 476dcb43f..0b91dcc79 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -1,12 +1,12 @@ package sqlancer; +import java.util.Objects; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.Randomly.StringGenerationStrategy; -import java.util.Objects; - @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { public static final int NO_SET_PORT = -1; From 679044ac8e6e570410a7440c95b08eb25b40caf4 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 10 Oct 2022 17:15:15 -0400 Subject: [PATCH 225/856] Update CockroachDB to v22.2.0-beta.4 --- .github/workflows/main.yml | 6 +++--- src/sqlancer/cockroachdb/CockroachDBBugs.java | 4 ++++ src/sqlancer/cockroachdb/CockroachDBErrors.java | 9 +++++++++ 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5d069d3be..348bdd976 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,11 +109,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v21.1.7.linux-amd64.tgz | tar xvz - cd cockroach-v21.1.7.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.0-beta.4.linux-amd64.tgz | tar xvz + cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v21.1.7.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 3a980a8c7..0d708c9a2 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -50,6 +50,10 @@ public final class CockroachDBBugs { // https://github.com/cockroachdb/cockroach/issues/85499 public static boolean bug85499 = true; + // https://github.com/cockroachdb/cockroach/issues/88037 + // TODO: This should be fixed in v22.2.1. + public static boolean bug88037 = true; + private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 66a5e5d7d..bbbda7baf 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -186,6 +186,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (CockroachDBBugs.bug85499) { errors.add("estimated row count must be non-zero"); } + if (CockroachDBBugs.bug88037) { + errors.add("expected required columns to be a subset of output columns"); + } errors.add("unable to vectorize execution plan"); // SET vectorize=experimental_always; errors.add(" mismatched physical types at index"); // SET vectorize=experimental_always; @@ -203,6 +206,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("ERROR: for SELECT DISTINCT, ORDER BY expressions must appear in select list"); addArrayErrors(errors); + addComputedColumnErrors(errors); } private static void addArrayErrors(ExpectedErrors errors) { @@ -308,4 +312,9 @@ public static void addTransactionErrors(ExpectedErrors errors) { errors.add("current transaction is aborted"); } + private static void addComputedColumnErrors(ExpectedErrors errors) { + // computed columns + errors.add("computed column expressions cannot reference computed columns"); + } + } From 4c20a94b3ad2c037e1a66c0b637184f8c20faa7e Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 20 Oct 2022 22:13:23 +0200 Subject: [PATCH 226/856] [ClickHouse] Simplify errors skip lists (#596) Simplify ClickHouse error skip lists --- src/sqlancer/clickhouse/ClickHouseErrors.java | 72 ++++++++----------- .../gen/ClickHouseTableGenerator.java | 2 +- .../tlp/ClickHouseTLPAggregateOracle.java | 1 - .../oracle/tlp/ClickHouseTLPBase.java | 1 - .../oracle/tlp/ClickHouseTLPHavingOracle.java | 2 - .../oracle/tlp/ClickHouseTLPWhereOracle.java | 1 - 6 files changed, 29 insertions(+), 50 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 908e5afec..5aa973d22 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -8,64 +8,48 @@ private ClickHouseErrors() { } public static void addExpectedExpressionErrors(ExpectedErrors errors) { - errors.add("Illegal type"); + errors.add("is not under aggregate function and not in GROUP BY"); errors.add("Argument at index 1 for function like must be constant"); errors.add("Argument at index 1 for function notLike must be constant"); - errors.add("does not return a value of type UInt8"); - errors.add("invalid escape sequence"); - errors.add("invalid character class range"); - errors.add("Memory limit"); - errors.add("There is no supertype for types"); + errors.add("argument of function"); + errors.add("Attempt to read after eof: while converting"); errors.add("Bad get: has Int64, requested UInt64"); + errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 errors.add("Cannot convert string"); - errors.add("Cannot read floating point value"); + errors.add("Cannot insert NULL value into a column of type"); errors.add("Cannot parse infinity."); - errors.add("Attempt to read after eof: while converting"); - errors.add( - "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); - errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database - errors.add("is not under aggregate function"); - errors.add("Invalid type for filter in"); - errors.add("argument of function"); - errors.add(" is not under aggregate function and not in GROUP BY"); - errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () - errors.add("OptimizedRegularExpression: cannot compile re2"); - errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 - errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation + errors.add("Cannot parse Int32 from String, because value is too short"); errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 errors.add("Cannot parse number with a sign character but without any numeric character"); errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); + errors.add("Cannot parse string"); + errors.add("Cannot read floating point value"); + errors.add("Cyclic aliases: default expression and column type are incompatible"); + errors.add("Directory for table data"); + errors.add("Directory not empty"); + errors.add("does not return a value of type UInt8"); + errors.add("does not return a value of type UInt8"); + errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database + errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); - errors.add("Positional argument out of bounds"); + errors.add("Illegal type"); errors.add("Illegal value (aggregate function) for positional argument in GROUP BY"); - errors.add("Invalid escape sequence at the end of LIKE pattern"); - } - - public static void addExpressionHavingErrors(ExpectedErrors errors) { - errors.add("Memory limit"); errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 - } - - public static void addQueryErrors(ExpectedErrors errors) { - errors.add("Memory limit"); - } - - public static void addGroupingErrors(ExpectedErrors errors) { - errors.add("Memory limit"); - } - - public static void addTableManipulationErrors(ExpectedErrors errors) { + errors.add("invalid character class range"); + errors.add("Invalid escape sequence at the end of LIKE pattern"); + errors.add("invalid escape sequence"); + errors.add("Invalid type for filter in"); + errors.add("is not under aggregate function"); + errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation + errors.add( + "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); errors.add("Memory limit"); - errors.add("Directory for table data"); - errors.add("Directory not empty"); + errors.add("OptimizedRegularExpression: cannot compile re2"); errors.add("Partition key cannot contain constants"); - errors.add("Cannot convert string"); - errors.add("argument of function"); - errors.add("Attempt to read after eof: while converting"); - errors.add("Sorting key cannot contain constants"); - errors.add("Sampling expression must be present in the primary key"); - errors.add("Cyclic aliases: default expression and column type are incompatible"); errors.add("Positional argument out of bounds"); + errors.add("Sampling expression must be present in the primary key"); + errors.add("Sorting key cannot contain constants"); + errors.add("There is no supertype for types"); } } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index 4d6f97008..c2b2e89c2 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -37,7 +37,7 @@ public static SQLQueryAdapter createTableStatement(String tableName, ClickHouseTableGenerator chTableGenerator = new ClickHouseTableGenerator(tableName, globalState); chTableGenerator.start(); ExpectedErrors errors = new ExpectedErrors(); - ClickHouseErrors.addTableManipulationErrors(errors); + ClickHouseErrors.addExpectedExpressionErrors(errors); return new SQLQueryAdapter(chTableGenerator.sb.toString(), errors, true); } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 7bf1f01a7..1538e0a5f 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -26,7 +26,6 @@ public class ClickHouseTLPAggregateOracle extends ClickHouseTLPBase { public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState state) { super(state); ClickHouseErrors.addExpectedExpressionErrors(errors); - ClickHouseErrors.addQueryErrors(errors); } @Override diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 045aadc51..d3cfc6b11 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -31,7 +31,6 @@ public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase Date: Sat, 22 Oct 2022 03:59:04 +0000 Subject: [PATCH 227/856] build(deps): bump slf4j-simple from 1.7.36 to 2.0.3 Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 1.7.36 to 2.0.3. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_1.7.36...v_2.0.3) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0b104497f..6dae29f62 100644 --- a/pom.xml +++ b/pom.xml @@ -325,7 +325,7 @@ org.slf4j slf4j-simple - 1.7.36 + 2.0.3 ru.yandex.clickhouse From 38b24ecd73a086147c5320d2527d520873e727e1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Oct 2022 12:15:01 +0800 Subject: [PATCH 228/856] Reference renamed SLF4 variable --- src/sqlancer/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a4404a6fa..a3f70b9f1 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -38,7 +38,7 @@ public final class Main { static boolean progressMonitorStarted; static { - System.setProperty(org.slf4j.impl.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "ERROR"); + System.setProperty(org.slf4j.simple.SimpleLogger.DEFAULT_LOG_LEVEL_KEY, "ERROR"); if (!LOG_DIRECTORY.exists()) { LOG_DIRECTORY.mkdir(); } From f32e0df931cc0a71e6cea1acb60124976e4ff9d7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Oct 2022 11:34:36 +0800 Subject: [PATCH 229/856] Remove the Travis CI references --- CONTRIBUTING.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 81177e352..e7567316f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,12 +50,12 @@ You can run them using the following command: mvn verify ``` -We use [Travis-CI](https://travis-ci.com/) to automatically check PRs. +We use [GitHub Actions](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) to automatically check PRs. ## Testing -As part of the Travis-CI gate, we use smoke testing by running SQLancer on each supported DBMS for some minutes, to test that nothing is obviously broken. For DBMS for which all bugs have been fixed, we verify that SQLancer cannot find any further bugs (i.e., the return code is zero). +As part of the GitHub Actions check, we use smoke testing by running SQLancer on each supported DBMS for some minutes, to test that nothing is obviously broken. For DBMS for which all bugs have been fixed, we verify that SQLancer cannot find any further bugs (i.e., the return code is zero). In addition, we use [unit tests](https://github.com/sqlancer/sqlancer/tree/master/test/sqlancer) to test SQLancer's core functionality, such as random string and number generation as well as option passing. When fixing a bug, add a unit test, if it is easily possible. @@ -65,13 +65,13 @@ You can run the tests using the following command: mvn test ``` -Note that per default, the smoke testing is performed only for embedded DBMS (i.e., DuckDB and SQLite). To run smoke tests also for the other DBMS, you need to set environment variables. For example, you can run the MySQL smoke testing (and no other tests) using the following command: +Note that per default, the smoke testing is performed only for embedded DBMS (e.g., DuckDB and SQLite). To run smoke tests also for the other DBMS, you need to set environment variables. For example, you can run the MySQL smoke testing (and no other tests) using the following command: ``` MYSQL_AVAILABLE=true mvn -Dtest=TestMySQL test ``` -For up-to-date testing commands, check out the `.travis.yml` file. +For up-to-date testing commands, check out the `.github/workflows/main.yml` file. ## Reviewing @@ -88,4 +88,4 @@ Please pay attention to good commit messages (in particular subject lines). As b 2. Do not end the subject line with a period. For example, write "Refactor the handling of indexes" rather than "Refactor the handling of indexes.". 3. Use the imperative mood in the subject line. For example, write "Refactor the handling of indexes" rather than "Refactoring" or "Refactor**ed** the handling of indexes". -Please also pay attention to a clean commit history. Rather than merging with the main branch, use `git rebase` to rebase your commits on the main branch. Sometimes, it might happen that you discover an issue only after having already created a commit, for example, when an issue is found by `mvn verify` in the Travis CI. Do not introduce a separate commit for such issues. If the issue was introduced by the last commit, you can fix the issue, and use `git commit --amend` to change the latest commit. If the change was introduced by one of the previous commits, you can use `git rebase -i` to change the respective commit. If you already have a number of such commits, you can use `git squash` to "collapse" multiple commits into one. For more information, you might want to read [How (and Why!) to Keep Your Git Commit History Clean](https://about.gitlab.com/blog/2018/06/07/keeping-git-commit-history-clean/) written by Kushal Pandya. +Please also pay attention to a clean commit history. Rather than merging with the main branch, use `git rebase` to rebase your commits on the main branch. Sometimes, it might happen that you discover an issue only after having already created a commit, for example, when an issue is found by `mvn verify` in the CI checks. Do not introduce a separate commit for such issues. If the issue was introduced by the last commit, you can fix the issue, and use `git commit --amend` to change the latest commit. If the change was introduced by one of the previous commits, you can use `git rebase -i` to change the respective commit. If you already have a number of such commits, you can use `git squash` to "collapse" multiple commits into one. For more information, you might want to read [How (and Why!) to Keep Your Git Commit History Clean](https://about.gitlab.com/blog/2018/06/07/keeping-git-commit-history-clean/) written by Kushal Pandya. From eabd7b1440cc299b93c7cb62f6e1b8269c57307f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 22 Oct 2022 11:51:32 +0800 Subject: [PATCH 230/856] Document how to deal with repeatedly triggered bugs --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e7567316f..b28a1be29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,6 +30,10 @@ For a permissive DBMS, implementing the expression generator is easier, since th For a strict DBMS, the better approach is typically to attempt to generate expressions of the expected type. For PostgreSQL, the expression generator thus expects an additional type argument (see [PostgreSQLExpressionGenerator](https://github.com/sqlancer/sqlancer/blob/86647df8aa2dd8d167b5c3ce3297290f5b0b2bcd/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java#L251)). This type is propagated recursively. For example, if we require a predicate for the `WHERE` clause, we pass boolean as a type. The expression generator then calls a method `generateBooleanExpression` that attempts to produce a boolean expression, by, for example, generating a comparison (e.g., `<=`). For the comparison's operands, a random type is then selected and propagated. For example, if an integer type is selected, then `generateExpression` is called with this type once for the left operand, and once for the right operand. Note that this process does not guarantee that the expression will indeed have the expected type. It might happen, for example, that the expression generator attempts to produce an integer value, but that it produces a double value instead, namely when an integer overflow occurs, which, depending on the DBMS, implicitly converts the result to a floating-point value. +### Unfixed Bugs + +Often, some bugs are fixed only after an extended period, meaning that SQLancer will repeatedly report the same bug. In such cases, it might be possible to avoid generating the problematic pattern, or adding an expected error with the internal error message. Rather than, for example, commenting out the code with the bug-inducing pattern, a pattern implemented by the [TiDBBugs class](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java) should be applied. The core idea is to use a public, static flag for each issue, which is set to true as long as the issue persists (e.g., see [bug35652](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java#L55)). The work-around code is then executed—or the problematic pattern should not be generated—if the flag is set to true (e.g., [an expected error is added for bug35652](https://github.com/sqlancer/sqlancer/blob/59564d818d991d54b32fa5a79c9f733799c090f2/src/sqlancer/tidb/TiDBErrors.java#L47)). This makes it easy to later on identify and remove all such work-around code once the issue has been fixed. + ## Options SQLancer uses [JCommander](https://jcommander.org/) for handling options. The `MainOptions` class contains options that are expected to be supported by all DBMS-testing implementations. Furthermore, each `*Provider` class provides a method to return an additional set of supported options. From 75c626f1e90d8bec871412f08d23d982c22de702 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 24 Oct 2022 14:46:34 +0200 Subject: [PATCH 231/856] [ClickHouse] Added NoRec Oracle for ClickHouse and other improvements (#598) * Do not print failed CREATE and INSERT statements in reproduction scenario * ClickHouse passes check_names.py * More errors ignored * null results are now correctly reported from JDBC driver * Enable JOINs as only 1 table was generated before. Add USE statement for easier repro * Currently no support for estimated result. Drop dead code * Support NoRec. Added Alias operation. Co-authored-by: Manuel Rigger --- src/check_names.py | 2 +- src/sqlancer/Main.java | 4 +- src/sqlancer/MainOptions.java | 7 + src/sqlancer/clickhouse/ClickHouseErrors.java | 32 +++-- .../clickhouse/ClickHouseOptions.java | 7 + .../clickhouse/ClickHouseProvider.java | 5 +- src/sqlancer/clickhouse/ClickHouseSchema.java | 77 +++++++---- .../clickhouse/ClickHouseToStringVisitor.java | 11 +- .../clickhouse/ClickHouseVisitor.java | 5 + .../ast/ClickHouseAliasOperation.java | 28 ++++ .../ast/ClickHouseColumnReference.java | 17 +-- .../clickhouse/ast/ClickHouseExpression.java | 23 ++-- .../gen/ClickHouseExpressionGenerator.java | 33 ++++- .../gen/ClickHouseTableGenerator.java | 2 +- .../oracle/norec/ClickHouseNoRECOracle.java | 125 ++++++++++++++++++ .../tlp/ClickHouseTLPAggregateOracle.java | 2 +- .../oracle/tlp/ClickHouseTLPBase.java | 4 +- .../tlp/ClickHouseTLPGroupByOracle.java | 4 +- .../oracle/tlp/ClickHouseTLPHavingOracle.java | 2 +- test/sqlancer/dbms/TestClickHouse.java | 31 +++-- 20 files changed, 334 insertions(+), 87 deletions(-) create mode 100644 src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java create mode 100644 src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java diff --git a/src/check_names.py b/src/check_names.py index f76b881ab..bca448b19 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -18,7 +18,7 @@ def verify_prefix(prefix, files): print('The class name of ' + f + ' does not start with ' + prefix) exit(-1) -# TODO: ClickHouse (wait for https://github.com/sqlancer/sqlancer/pull/39) +verify_prefix('ClickHouse', get_java_files("sqlancer/clickhouse/")) verify_prefix('CockroachDB', get_java_files("sqlancer/cockroachdb/")) verify_prefix('DuckDB', get_java_files("sqlancer/duckdb")) verify_prefix('MariaDB', get_java_files("sqlancer/mariadb/")) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index a3f70b9f1..7953386b7 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -222,10 +222,12 @@ public static class QueryManager { } public boolean execute(Query q, String... fills) throws Exception { - globalState.getState().logStatement(q); boolean success; success = q.execute(globalState, fills); Main.nrSuccessfulActions.addAndGet(1); + if (globalState.getOptions().loggerPrintFailed() || success) { + globalState.getState().logStatement(q); + } return success; } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 0b91dcc79..2916dbdd2 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -47,6 +47,9 @@ public class MainOptions { @Parameter(names = "--log-execution-time", description = "Logs the execution time of each statement (requires --log-each-select to be enabled)", arity = 1) private boolean logExecutionTime = true; // NOPMD + @Parameter(names = "--print-failed", description = "Logs failed insert, create and other statements without results", arity = 1) + private boolean loggerPrintFailed = true; // NOPMD + @Parameter(names = "--username", description = "The user name used to log into the DBMS") private String userName = "sqlancer"; // NOPMD @@ -141,6 +144,10 @@ public boolean logExecutionTime() { return logExecutionTime; } + public boolean loggerPrintFailed() { + return loggerPrintFailed; + } + public int getNrQueries() { return nrQueries; } diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 5aa973d22..88597d681 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -8,18 +8,15 @@ private ClickHouseErrors() { } public static void addExpectedExpressionErrors(ExpectedErrors errors) { - errors.add("is not under aggregate function and not in GROUP BY"); errors.add("Argument at index 1 for function like must be constant"); errors.add("Argument at index 1 for function notLike must be constant"); - errors.add("argument of function"); errors.add("Attempt to read after eof: while converting"); errors.add("Bad get: has Int64, requested UInt64"); - errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 errors.add("Cannot convert string"); errors.add("Cannot insert NULL value into a column of type"); - errors.add("Cannot parse infinity."); errors.add("Cannot parse Int32 from String, because value is too short"); errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 + errors.add("Cannot parse infinity."); errors.add("Cannot parse number with a sign character but without any numeric character"); errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); errors.add("Cannot parse string"); @@ -27,22 +24,12 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Cyclic aliases: default expression and column type are incompatible"); errors.add("Directory for table data"); errors.add("Directory not empty"); - errors.add("does not return a value of type UInt8"); - errors.add("does not return a value of type UInt8"); - errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); errors.add("Illegal type"); errors.add("Illegal value (aggregate function) for positional argument in GROUP BY"); - errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 - errors.add("invalid character class range"); errors.add("Invalid escape sequence at the end of LIKE pattern"); - errors.add("invalid escape sequence"); errors.add("Invalid type for filter in"); - errors.add("is not under aggregate function"); - errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation - errors.add( - "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); errors.add("Memory limit"); errors.add("OptimizedRegularExpression: cannot compile re2"); errors.add("Partition key cannot contain constants"); @@ -50,6 +37,23 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Sampling expression must be present in the primary key"); errors.add("Sorting key cannot contain constants"); errors.add("There is no supertype for types"); + errors.add("argument of function"); + errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 + errors.add("but its arguments considered equal according to constraints"); + errors.add("does not return a value of type UInt8"); + errors.add("does not return a value of type UInt8"); + errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database + errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 + errors.add("invalid character class range"); + errors.add("invalid escape sequence"); + errors.add("is not under aggregate function and not in GROUP BY"); + errors.add("is not under aggregate function"); + errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation + errors.add( + "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); + errors.add("there are only columns"); + errors.add("there are columns"); + errors.add("in block. (NOT_FOUND_COLUMN_IN_BLOCK)"); } } diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index c58697c98..8a2056c1b 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -11,6 +11,7 @@ import sqlancer.OracleFactory; import sqlancer.clickhouse.ClickHouseOptions.ClickHouseOracleFactory; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.oracle.norec.ClickHouseNoRECOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; @@ -60,6 +61,12 @@ public TestOracle create(ClickHouseGlobalState globalState) throws SQLException public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPHavingOracle(globalState); } + }, + NoREC { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseNoRECOracle(globalState); + } }; } diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 7ff2de11b..e3a1fb21c 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -87,7 +87,7 @@ protected ClickHouseSchema readSchema() throws SQLException { @Override public void generateDatabase(ClickHouseGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(1); i++) { + for (int i = 0; i < Randomly.fromOptions(5); i++) { boolean success; do { String tableName = ClickHouseCommon.createTableName(i); @@ -96,6 +96,7 @@ public void generateDatabase(ClickHouseGlobalState globalState) throws Exception } while (!success); } + // TODO: add more Actions to populate table StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), ClickHouseProvider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { @@ -126,6 +127,8 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ globalState.getState().logStatement(dropDatabaseCommand); String createDatabaseCommand = "CREATE DATABASE IF NOT EXISTS " + databaseName; globalState.getState().logStatement(createDatabaseCommand); + String useDatabaseCommand = "USE " + databaseName; // Noop. To reproduce easier. + globalState.getState().logStatement(useDatabaseCommand); try (Statement s = con.createStatement()) { s.execute(dropDatabaseCommand); Thread.sleep(1000); diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index 21b39e021..ff58fc7a9 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -60,14 +60,15 @@ public static class ClickHouseColumn extends AbstractTableColumn getTableColumns(SQLConnection con, String boolean isAlias = "ALIAS".compareTo(defaultType) == 0; boolean isMaterialized = "MATERIALIZED".compareTo(defaultType) == 0; ClickHouseColumn c = new ClickHouseColumn(columnName, getColumnType(dataType), isAlias, - isMaterialized); + isMaterialized, null); columns.add(c); } } diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index beaa37b49..2d78fcc8c 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; @@ -129,7 +130,8 @@ public void visit(ClickHouseCastOperation cast) { @Override public void visit(ClickHouseExpression.ClickHouseJoin join) { - + sb.append("ON "); + visit(join.getOnClause()); } @Override @@ -141,6 +143,13 @@ public void visit(ClickHouseColumnReference c) { } } + @Override + public void visit(ClickHouseAliasOperation alias) { + visit(alias.getExpression()); + sb.append(" AS "); + sb.append(alias.getAlias()); + } + public static String asString(ClickHouseExpression expr) { ClickHouseToStringVisitor visitor = new ClickHouseToStringVisitor(); visitor.visit(expr); diff --git a/src/sqlancer/clickhouse/ClickHouseVisitor.java b/src/sqlancer/clickhouse/ClickHouseVisitor.java index e1473f669..93a46d607 100644 --- a/src/sqlancer/clickhouse/ClickHouseVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseVisitor.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; @@ -51,6 +52,8 @@ default void visit(ClickHouseExpression.ClickHousePostfixText op) { void visit(ClickHouseCastOperation cast); + void visit(ClickHouseAliasOperation alias); + void visit(ClickHouseExpression.ClickHouseJoin join); void visit(ClickHouseAggregate aggregate); @@ -78,6 +81,8 @@ default void visit(ClickHouseExpression expr) { visit((ClickHouseExpression.ClickHousePostfixText) expr); } else if (expr instanceof ClickHouseAggregate) { visit((ClickHouseAggregate) expr); + } else if (expr instanceof ClickHouseAliasOperation) { + visit((ClickHouseAliasOperation) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java new file mode 100644 index 000000000..b7976ff46 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseAliasOperation.java @@ -0,0 +1,28 @@ +package sqlancer.clickhouse.ast; + +public class ClickHouseAliasOperation extends ClickHouseExpression { + + private final ClickHouseExpression expression; + private final String alias; + + public ClickHouseAliasOperation(ClickHouseExpression expression, String alias) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.alias = alias; + } + + @Override + public ClickHouseConstant getExpectedValue() { + return expression.getExpectedValue(); + } + + public ClickHouseExpression getExpression() { + return expression; + } + + public String getAlias() { + return alias; + } +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java index 818da0e7e..796eb8333 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java @@ -5,28 +5,17 @@ public class ClickHouseColumnReference extends ClickHouseExpression { private final ClickHouseColumn column; - private final ClickHouseConstant value; - public ClickHouseColumnReference(ClickHouseColumn column, ClickHouseConstant value) { + public ClickHouseColumnReference(ClickHouseColumn column) { this.column = column; - this.value = value; } - public static ClickHouseColumnReference create(ClickHouseColumn column, ClickHouseConstant value) { - return new ClickHouseColumnReference(column, value); + public static ClickHouseColumnReference create(ClickHouseColumn column) { + return new ClickHouseColumnReference(column); } public ClickHouseColumn getColumn() { return column; } - public ClickHouseConstant getValue() { - return value; - } - - @Override - public ClickHouseConstant getExpectedValue() { - return value; - } - } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index c3642c208..161fd0206 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -36,25 +36,28 @@ public ClickHouseExpression getExpression() { } public static class ClickHouseJoin extends ClickHouseExpression { - // TODO: support ANY, ALL, ASOF modifiers public enum JoinType { INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, NATURAL, LEFT_SEMI, RIGHT_SEMI, LEFT_ANTI, RIGHT_ANTI; } - private final ClickHouseSchema.ClickHouseTable table; + private final ClickHouseSchema.ClickHouseTable leftTable; + private final ClickHouseSchema.ClickHouseTable rightTable; private ClickHouseExpression onClause; private final ClickHouseJoin.JoinType type; - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable table, ClickHouseExpression onClause, - ClickHouseJoin.JoinType type) { - this.table = table; + public ClickHouseJoin(ClickHouseSchema.ClickHouseTable leftTable, ClickHouseSchema.ClickHouseTable rightTable, + ClickHouseJoin.JoinType type, ClickHouseExpression onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; this.onClause = onClause; this.type = type; } - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable table, ClickHouseJoin.JoinType type) { - this.table = table; + public ClickHouseJoin(ClickHouseSchema.ClickHouseTable leftTable, ClickHouseSchema.ClickHouseTable rightTable, + ClickHouseJoin.JoinType type) { + this.leftTable = leftTable; + this.rightTable = rightTable; if (type != ClickHouseJoin.JoinType.NATURAL) { throw new AssertionError(); } @@ -62,8 +65,12 @@ public ClickHouseJoin(ClickHouseSchema.ClickHouseTable table, ClickHouseJoin.Joi this.type = type; } + public ClickHouseSchema.ClickHouseTable getLeftTable() { + return leftTable; + } + public ClickHouseSchema.ClickHouseTable getTable() { - return table; + return rightTable; } public ClickHouseExpression getOnClause() { diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 1054055f9..6e86b28ec 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -71,6 +71,14 @@ protected ClickHouseExpression generateExpression(ClickHouseLancerDataType type, } } + protected ClickHouseExpression generateJoinClause(ClickHouseSchema.ClickHouseTable leftTable, + ClickHouseSchema.ClickHouseTable rightTable) { + ClickHouseExpression leftExpr = getColumnNameFromTable(leftTable); + ClickHouseExpression rightExpr = getColumnNameFromTable(rightTable); + return new ClickHouseBinaryComparisonOperation(leftExpr, rightExpr, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.EQUALS); + } + @Override protected ClickHouseExpression generateColumn(ClickHouseLancerDataType type) { if (columns.isEmpty()) { @@ -80,7 +88,20 @@ protected ClickHouseExpression generateColumn(ClickHouseLancerDataType type) { .filter(c -> c.getType().getType().name().equals(type.getType().name())).collect(Collectors.toList()); ClickHouseColumn column = filteredColumns.isEmpty() ? Randomly.fromList(columns) : Randomly.fromList(filteredColumns); - return new ClickHouseColumnReference(column, null); + return new ClickHouseColumnReference(column); + } + + protected ClickHouseExpression getColumnNameFromTable(ClickHouseSchema.ClickHouseTable table) { + if (columns.isEmpty()) { + return generateConstant(ClickHouseLancerDataType.getRandom()); + } + List filteredColumns = columns.stream().filter(c -> c.getTable() == table) + .collect(Collectors.toList()); + if (filteredColumns.isEmpty()) { + return generateConstant(ClickHouseLancerDataType.getRandom()); + } + ClickHouseColumn column = Randomly.fromList(filteredColumns); + return new ClickHouseColumnReference(column); } @Override @@ -97,17 +118,17 @@ public List getRandomJoinClauses( if (Randomly.getBoolean() && tables.size() > 1) { int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); for (int i = 0; i < nrJoinClauses; i++) { - ClickHouseExpression joinClause = generateExpression(ClickHouseLancerDataType.getRandom()); - ClickHouseSchema.ClickHouseTable table = Randomly.fromList(tables); - tables.remove(table); + ClickHouseSchema.ClickHouseTable leftTable = Randomly.fromList(tables); + ClickHouseSchema.ClickHouseTable rightTable = Randomly.fromList(tables); + ClickHouseExpression joinClause = generateJoinClause(leftTable, rightTable); ClickHouseExpression.ClickHouseJoin.JoinType options; options = Randomly.fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); if (options == ClickHouseExpression.ClickHouseJoin.JoinType.NATURAL) { // NATURAL joins do not have an ON clause joinClause = null; } - ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(table, joinClause, - options); + ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(leftTable, rightTable, + options, joinClause); joinStatements.add(j); } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index c2b2e89c2..5b62e88bc 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -55,7 +55,7 @@ public void start() { sb.append(" ("); int nrColumns = 1 + Randomly.smallNumber(); for (int i = 0; i < nrColumns; i++) { - columns.add(ClickHouseSchema.ClickHouseColumn.createDummy(ClickHouseCommon.createColumnName(i))); + columns.add(ClickHouseSchema.ClickHouseColumn.createDummy(ClickHouseCommon.createColumnName(i), null)); } for (int i = 0; i < nrColumns; i++) { if (i != 0) { diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java new file mode 100644 index 000000000..b8da925e5 --- /dev/null +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -0,0 +1,125 @@ +package sqlancer.clickhouse.oracle.norec; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.clickhouse.ClickHouseErrors; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; +import sqlancer.clickhouse.ClickHouseToStringVisitor; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; +import sqlancer.clickhouse.ast.ClickHouseColumnReference; +import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.ast.ClickHouseTableReference; +import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; + +public class ClickHouseNoRECOracle extends NoRECBase implements TestOracle { + + private final ClickHouseSchema s; + + public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + ClickHouseErrors.addExpectedExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); + ClickHouseExpression randomWhereCondition = gen.generateExpression(ClickHouseLancerDataType.getRandom()); + List tables = randomTables.getTables(); + List tableList = tables.stream().map(t -> new ClickHouseTableReference(t)) + .collect(Collectors.toList()); + List joins = gen.getRandomJoinClauses(tables); + int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); + int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, + randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getSecondQuery(List tableList, ClickHouseExpression whereClause, + List joins) throws SQLException { + ClickHouseSelect select = new ClickHouseSelect(); + + ClickHouseExpression inner = new ClickHouseAliasOperation(whereClause, "check"); + + select.setFetchColumns(Arrays.asList(inner)); + select.setFromList(tableList); + select.setJoinClauses(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(check <> 0) FROM (" + ClickHouseToStringVisitor.asString(select) + + ") as res"; + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(SQLConnection con, List tableList, + List columns, ClickHouseExpression randomWhereCondition, + List joins) throws SQLException { + ClickHouseSelect select = new ClickHouseSelect(); + List allColumns = columns.stream().map((c) -> new ClickHouseColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions( + new ClickHouseExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + select.setJoinClauses(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = ClickHouseToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 1538e0a5f..220b8f6c0 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -76,7 +76,7 @@ public void check() throws SQLException { throw new AssertionError(); } else if (firstResult.isEmpty()) { return; - } else if (firstResult.size() == 1) { + } else if (firstResult.size() == 1 && secondResult.size() == 1) { if (firstResult.get(0).equals(secondResult.get(0))) { return; } else if (!ComparatorHelper.isEqualDouble(firstResult.get(0), secondResult.get(0))) { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index d3cfc6b11..245343830 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -50,8 +50,8 @@ public void check() throws SQLException { } List generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new ClickHouseColumnReference(c)) + .collect(Collectors.toList()); } @Override diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index bb3629604..0cdee3864 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -43,8 +43,8 @@ public void check() throws SQLException { @Override List generateFetchColumns() { List columns; - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new ClickHouseColumnReference(c)) + .collect(Collectors.toList()); return columns; } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index d015d8092..c7bde4b70 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -31,7 +31,7 @@ public void check() throws SQLException { ClickHouseSchema s = state.getSchema(); ClickHouseSchema.ClickHouseTables targetTables = s.getRandomTableNonEmptyTables(); List groupByColumns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c, null)).collect(Collectors.toList()); + .map(c -> new ClickHouseColumnReference(c)).collect(Collectors.toList()); List columns = targetTables.getColumns(); ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); ClickHouseExpressionGenerator aggrGen = new ClickHouseExpressionGenerator(state).allowAggregates(true) diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 2cf53eef2..073fdcbd3 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -35,8 +35,9 @@ public void testClickHouseTLPHaving() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPHaving")); + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "clickhouse", "--oracle", "TLPHaving")); } @Test @@ -45,8 +46,9 @@ public void testClickHouseTLPGroupBy() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPGroupBy")); + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "clickhouse", "--oracle", "TLPGroupBy")); } @Test @@ -55,8 +57,9 @@ public void testClickHouseTLPDistinct() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPDistinct")); + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "clickhouse", "--oracle", "TLPDistinct")); } @Test @@ -65,8 +68,20 @@ public void testClickHouseTLPAggregate() { boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); assertEquals(0, - Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPAggregate")); + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "clickhouse", "--oracle", "TLPAggregate")); + } + + @Test + public void testClickHouseNoREC() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "clickhouse", "--oracle", "NoREC")); } } From f2f1a99aeae389fffccf2640aab39d527b0fdba2 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Tue, 25 Oct 2022 12:34:42 +0200 Subject: [PATCH 232/856] fix (#601) --- .../clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 220b8f6c0..6275b6a87 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -74,7 +74,7 @@ public void check() throws SQLException { if (firstResult.size() != secondResult.size()) { throw new AssertionError(); - } else if (firstResult.isEmpty()) { + } else if (firstResult.isEmpty() || firstResult.equals(secondResult)) { return; } else if (firstResult.size() == 1 && secondResult.size() == 1) { if (firstResult.get(0).equals(secondResult.get(0))) { From 83f8bfb7991f16c39e70412000b8629f404aff1b Mon Sep 17 00:00:00 2001 From: Suri Zhang Date: Sun, 23 Oct 2022 20:54:45 +0800 Subject: [PATCH 233/856] Implement generators and ast for questdb --- src/sqlancer/questdb/QuestDBBugs.java | 10 ++ src/sqlancer/questdb/QuestDBErrors.java | 22 ++- src/sqlancer/questdb/QuestDBOptions.java | 12 +- src/sqlancer/questdb/QuestDBProvider.java | 62 +++++++- src/sqlancer/questdb/QuestDBSchema.java | 50 +++--- .../questdb/QuestDBToStringVisitor.java | 28 ++++ src/sqlancer/questdb/ast/QuestDBConstant.java | 89 +++++++++++ .../questdb/ast/QuestDBExpression.java | 4 + src/sqlancer/questdb/ast/QuestDBSelect.java | 16 ++ .../gen/QuestDBAlterIndexGenerator.java | 53 +++++++ .../questdb/gen/QuestDBDeleteGenerator.java | 15 -- .../gen/QuestDBExpressionGenerator.java | 146 ++++++++++++++++++ .../questdb/gen/QuestDBIndexGenerator.java | 14 -- .../questdb/gen/QuestDBInsertGenerator.java | 43 ++++-- .../questdb/gen/QuestDBTableGenerator.java | 33 +++- .../questdb/gen/QuestDBTruncateGenerator.java | 22 +++ .../test/QuestDBQueryPartitioningBase.java | 80 ++++++++++ .../QuestDBQueryPartitioningWhereTester.java | 41 +++++ 18 files changed, 656 insertions(+), 84 deletions(-) create mode 100644 src/sqlancer/questdb/QuestDBBugs.java create mode 100644 src/sqlancer/questdb/QuestDBToStringVisitor.java create mode 100644 src/sqlancer/questdb/ast/QuestDBConstant.java create mode 100644 src/sqlancer/questdb/ast/QuestDBExpression.java create mode 100644 src/sqlancer/questdb/ast/QuestDBSelect.java create mode 100644 src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java delete mode 100644 src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java delete mode 100644 src/sqlancer/questdb/gen/QuestDBIndexGenerator.java create mode 100644 src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java create mode 100644 src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java create mode 100644 src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java diff --git a/src/sqlancer/questdb/QuestDBBugs.java b/src/sqlancer/questdb/QuestDBBugs.java new file mode 100644 index 000000000..bbba765a3 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBBugs.java @@ -0,0 +1,10 @@ +package sqlancer.questdb; + +public final class QuestDBBugs { + + // https://github.com/questdb/questdb/issues/2689 + public static boolean bug2689 = true; + + private QuestDBBugs() { + } +} diff --git a/src/sqlancer/questdb/QuestDBErrors.java b/src/sqlancer/questdb/QuestDBErrors.java index 534c5c41d..44dbea148 100644 --- a/src/sqlancer/questdb/QuestDBErrors.java +++ b/src/sqlancer/questdb/QuestDBErrors.java @@ -1,4 +1,24 @@ package sqlancer.questdb; -public class QuestDBErrors { +import sqlancer.common.query.ExpectedErrors; + +public final class QuestDBErrors { + + private QuestDBErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + // TODO (anxing) + } + + public static void addGroupByErrors(ExpectedErrors errors) { + // TODO (anxing) + } + + public static void addInsertErrors(ExpectedErrors errors) { + // TODO (anxing) + errors.add("Invalid column"); + errors.add("inconvertible types:"); + errors.add("inconvertible value:"); + } } diff --git a/src/sqlancer/questdb/QuestDBOptions.java b/src/sqlancer/questdb/QuestDBOptions.java index ad2b593af..527e80cc8 100644 --- a/src/sqlancer/questdb/QuestDBOptions.java +++ b/src/sqlancer/questdb/QuestDBOptions.java @@ -12,6 +12,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.questdb.QuestDBOptions.QuestDBOracleFactory; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.test.QuestDBQueryPartitioningWhereTester; @Parameters(separators = "=", commandDescription = "QuestDB (default port: " + QuestDBOptions.DEFAULT_PORT + " default host: " + QuestDBOptions.DEFAULT_HOST + ")") @@ -20,17 +21,16 @@ public class QuestDBOptions implements DBMSSpecificOptions public static final int DEFAULT_PORT = 8812; @Parameter(names = "--oracle") - public QuestDBOracleFactory oracle = QuestDBOracleFactory.WHERE; + public List oracle = Arrays.asList(QuestDBOracleFactory.WHERE); public enum QuestDBOracleFactory implements OracleFactory { - // TODO: implement test oracles + // TODO (anxing): implement test oracles WHERE { @Override public TestOracle create(QuestDBGlobalState globalState) throws SQLException { - return null; + return new QuestDBQueryPartitioningWhereTester(globalState); } - }; - + } } @Parameter(names = "--username", description = "The user name used to log into QuestDB") @@ -41,7 +41,7 @@ public TestOracle create(QuestDBGlobalState globalState) throws SQLException { @Override public List getTestOracleFactory() { - return Arrays.asList(oracle); + return oracle; } public String getUserName() { diff --git a/src/sqlancer/questdb/QuestDBProvider.java b/src/sqlancer/questdb/QuestDBProvider.java index 997ab40da..d9932c8a3 100644 --- a/src/sqlancer/questdb/QuestDBProvider.java +++ b/src/sqlancer/questdb/QuestDBProvider.java @@ -10,16 +10,19 @@ import sqlancer.AbstractAction; import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; -import sqlancer.questdb.gen.QuestDBDeleteGenerator; -import sqlancer.questdb.gen.QuestDBIndexGenerator; +import sqlancer.questdb.gen.QuestDBAlterIndexGenerator; import sqlancer.questdb.gen.QuestDBInsertGenerator; import sqlancer.questdb.gen.QuestDBTableGenerator; +import sqlancer.questdb.gen.QuestDBTruncateGenerator; @AutoService(DatabaseProvider.class) public class QuestDBProvider extends SQLProviderAdapter { @@ -29,9 +32,9 @@ public QuestDBProvider() { public enum Action implements AbstractAction { INSERT(QuestDBInsertGenerator::getQuery), // - CREATE_INDEX(QuestDBIndexGenerator::getQuery), // - DELETE(QuestDBDeleteGenerator::generate); // - // TODO: maybe implement these later + ALTER_INDEX(QuestDBAlterIndexGenerator::getQuery), // + TRUNCATE(QuestDBTruncateGenerator::generate); // + // TODO (anxing): maybe implement these later // UPDATE(QuestDBUpdateGenerator::getQuery), // // CREATE_VIEW(QuestDBViewGenerator::generate), // @@ -47,6 +50,20 @@ public SQLQueryAdapter getQuery(QuestDBGlobalState state) throws Exception { } } + private static int mapActions(QuestDBGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case ALTER_INDEX: + return r.getInteger(0, 3); + case TRUNCATE: + return r.getInteger(0, 5); + default: + throw new AssertionError("Unknown action: " + a); + } + } + public static class QuestDBGlobalState extends SQLGlobalState { @Override @@ -58,7 +75,23 @@ protected QuestDBSchema readSchema() throws SQLException { @Override public void generateDatabase(QuestDBGlobalState globalState) throws Exception { - // TODO: should follow duckdb or cockrachdb? what's the difference between generate and create db? + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new QuestDBTableGenerator().getQuery(globalState, null); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + QuestDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); } @Override @@ -84,15 +117,30 @@ public SQLConnection createDatabase(QuestDBGlobalState globalState) throws Excep Connection con = DriverManager.getConnection(url, properties); // QuestDB cannot create or drop `DATABASE`, can only create or drop `TABLE` globalState.getState().logStatement("DROP TABLE IF EXISTS " + tableName + " CASCADE"); - SQLQueryAdapter createTableCommand = new QuestDBTableGenerator().getQuery(globalState); + SQLQueryAdapter createTableCommand = new QuestDBTableGenerator().getQuery(globalState, tableName); globalState.getState().logStatement(createTableCommand); + globalState.getState().logStatement("DROP TABLE IF EXISTS " + tableName); try (Statement s = con.createStatement()) { s.execute("DROP TABLE IF EXISTS " + tableName); } + // TODO(anxing): Drop all previous tables in db + // List tableNames = + // globalState.getSchema().getDatabaseTables().stream().map(AbstractTable::getName).collect(Collectors.toList()); + // for (String tName : tableNames) { + // try (Statement s = con.createStatement()) { + // String query = "DROP TABLE IF EXISTS " + tName; + // globalState.getState().logStatement(query); + // s.execute(query); + // } + // } try (Statement s = con.createStatement()) { s.execute(createTableCommand.getQueryString()); } + // drop test table + try (Statement s = con.createStatement()) { + s.execute("DROP TABLE IF EXISTS " + tableName); + } con.close(); con = DriverManager.getConnection(url, properties); return new SQLConnection(con); diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index ef2026706..0b5563a31 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -25,11 +25,11 @@ public class QuestDBSchema extends AbstractSchema getTableNames(SQLConnection con) throws SQLException { + protected static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery("SHOW TABLES;")) { @@ -277,7 +277,7 @@ private static List getTableNames(SQLConnection con) throws SQLException String tName = rs.getString("table"); // exclude reserved tables for testing if (!QuestDBTables.RESERVED_TABLES.contains(tName)) { - tableNames.add(rs.getString("table")); + tableNames.add(tName); } } } @@ -298,10 +298,6 @@ private static List getTableColumns(SQLConnection con, String tab } } } - if (columns.stream().noneMatch(c -> c.isIndexed())) { - // TODO: implement an option to enable/disable rowids - columns.add(new QuestDBColumn("rowid", new QuestDBCompositeDataType(QuestDBDataType.INT, 4), false)); - } return columns; } diff --git a/src/sqlancer/questdb/QuestDBToStringVisitor.java b/src/sqlancer/questdb/QuestDBToStringVisitor.java new file mode 100644 index 000000000..7552b06cb --- /dev/null +++ b/src/sqlancer/questdb/QuestDBToStringVisitor.java @@ -0,0 +1,28 @@ +package sqlancer.questdb; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.questdb.ast.QuestDBConstant; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof QuestDBConstant) { + visit((QuestDBConstant) expr); + } else { // TODO: maybe implement QuestDBSelect & Join + throw new AssertionError("Unknown class: " + expr.getClass()); + } + } + + private void visit(QuestDBConstant constant) { + sb.append(constant.toString()); + } + + public static String asString(Node expr) { + QuestDBToStringVisitor visitor = new QuestDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBConstant.java b/src/sqlancer/questdb/ast/QuestDBConstant.java new file mode 100644 index 000000000..e0950e656 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBConstant.java @@ -0,0 +1,89 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.newast.Node; + +public class QuestDBConstant implements Node { + private QuestDBConstant() { + } + + public static class QuestDBNullConstant extends QuestDBConstant { + @Override + public String toString() { + return "NULL"; + } + } + + public static class QuestDBIntConstant extends QuestDBConstant { + private final long value; + + public QuestDBIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + } + + public static class QuestDBBooleanConstant extends QuestDBConstant { + private final boolean value; + + public QuestDBBooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public boolean getValue() { + return value; + } + } + + public static Node createIntConstant(long val) { + return new QuestDBIntConstant(val); + } + + public static class QuestDBDoubleConstant extends QuestDBConstant { + + private final double value; + + public QuestDBDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + } + + public static Node createBooleanConstant(boolean val) { + return new QuestDBBooleanConstant(val); + } + + public static Node createNullConstant() { + return new QuestDBNullConstant(); + } + + public static Node createFloatConstant(double val) { + return new QuestDBDoubleConstant(val); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBExpression.java b/src/sqlancer/questdb/ast/QuestDBExpression.java new file mode 100644 index 000000000..c2c7f1d74 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBExpression.java @@ -0,0 +1,4 @@ +package sqlancer.questdb.ast; + +public interface QuestDBExpression { +} diff --git a/src/sqlancer/questdb/ast/QuestDBSelect.java b/src/sqlancer/questdb/ast/QuestDBSelect.java new file mode 100644 index 000000000..023384487 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBSelect.java @@ -0,0 +1,16 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class QuestDBSelect extends SelectBase> implements Node { + private boolean isDistinct; + + public void setDistinct(boolean distinct) { + isDistinct = distinct; + } + + public boolean isDistinct() { + return isDistinct; + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java new file mode 100644 index 000000000..1e8242148 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.questdb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; + +public final class QuestDBAlterIndexGenerator { + private QuestDBAlterIndexGenerator() { + } + + enum Action { + ADD_INDEX, DROP_INDEX + } + + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add(" does not have a column with name \"rowid\""); + errors.add("Table does not contain column rowid referenced in alter statement"); + errors.add("cannot create index"); + errors.add("Index flag is only supported for SYMBOL"); + errors.add("Invalid column: "); + + StringBuilder sb = new StringBuilder("ALTER TABLE "); + + QuestDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append(" "); + + sb.append("ALTER COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_INDEX: + sb.append("ADD INDEX"); + errors.add("already exists!"); + + break; + case DROP_INDEX: + sb.append("DROP INDEX"); + errors.add("Column is not indexed"); + break; + default: + throw new AssertionError("Unkown action:" + action); + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java b/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java deleted file mode 100644 index 8cfefd093..000000000 --- a/src/sqlancer/questdb/gen/QuestDBDeleteGenerator.java +++ /dev/null @@ -1,15 +0,0 @@ -package sqlancer.questdb.gen; - -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; - -public final class QuestDBDeleteGenerator { - private QuestDBDeleteGenerator() { - - } - - public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { - // TODO - return null; - } -} diff --git a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java new file mode 100644 index 000000000..82f82ac88 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java @@ -0,0 +1,146 @@ +package sqlancer.questdb.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBDataType; +import sqlancer.questdb.ast.QuestDBConstant; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBExpressionGenerator extends UntypedExpressionGenerator, QuestDBColumn> { + + private final QuestDBGlobalState globalState; + + public QuestDBExpressionGenerator(QuestDBGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, QuestDBUnaryPrefixOperator.NOT); + } + + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, QuestDBUnaryPostfixOperator.IS_NULL); + } + + @Override + public Node generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return QuestDBConstant.createNullConstant(); + } + QuestDBDataType type = QuestDBDataType.getRandomWithoutNull(); + switch (type) { + case INT: + return QuestDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + case BOOLEAN: + return QuestDBConstant.createBooleanConstant(Randomly.getBoolean()); + case FLOAT: + return QuestDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); + // case CHAR: + // case DATE: + // case TIMESTAMP: + // throw new IgnoreMeException(); + default: + throw new AssertionError("Unknown type: " + type); + } + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } else { + throw new AssertionError("Expression generation failed, depth=" + depth); + } + } + + @Override + protected Node generateColumn() { + QuestDBColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode<>(column); + } + + public enum QuestDBUnaryPostfixOperator implements Operator { + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + QuestDBUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static QuestDBUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum QuestDBUnaryPrefixOperator implements Operator { + + NOT("NOT"); + + private String textRepr; + + QuestDBUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static QuestDBUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum QuestDBBinaryLogicalOperator implements Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum QuestDBBinaryComparisonOperator implements Operator { + EQUALS("="), GREATER_THAN(">"), GREATER_THAN_EQUALS(">="), LESS_THAN("<"), SMALLER_THAN_EQUALS("<="), + NOT_EQUALS("!="); + + private String textRepr; + + QuestDBBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java b/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java deleted file mode 100644 index b994c22c0..000000000 --- a/src/sqlancer/questdb/gen/QuestDBIndexGenerator.java +++ /dev/null @@ -1,14 +0,0 @@ -package sqlancer.questdb.gen; - -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; - -public final class QuestDBIndexGenerator { - private QuestDBIndexGenerator() { - } - - public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { - // TODO - return null; - } -} diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java index 807917b46..754a0d2e5 100644 --- a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -1,27 +1,40 @@ package sqlancer.questdb.gen; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.questdb.QuestDBErrors; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; -import sqlancer.questdb.QuestDBSchema; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; +import sqlancer.questdb.QuestDBToStringVisitor; public class QuestDBInsertGenerator extends AbstractInsertGenerator { private final QuestDBGlobalState globalState; - // uncomment later: - // private final ExpectedErrors errors = new ExpectedErrors(); + private final ExpectedErrors errors = new ExpectedErrors(); public QuestDBInsertGenerator(QuestDBGlobalState globalState) { this.globalState = globalState; } private SQLQueryAdapter generate() { - // TODO: below is dummy implementation to pass compilation - QuestDBSchema.QuestDBTable table = globalState.getSchema().getRandomTable(); - table.getRandomNonEmptyColumnSubset(); - return null; + sb.append("INSERT INTO "); + QuestDBTable table = globalState.getSchema().getRandomTable(); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + QuestDBErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); } public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { @@ -29,7 +42,19 @@ public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { } @Override - protected void insertValue(QuestDBColumn tiDBColumn) { - // TODO + protected void insertColumns(List columns) { + sb.append("("); + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + insertValue(columns.get(nrColumn)); + } + sb.append(")"); + } + + @Override + protected void insertValue(QuestDBColumn questDBColumn) { + sb.append(QuestDBToStringVisitor.asString(new QuestDBExpressionGenerator(globalState).generateConstant())); } } diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java index 360c9169b..abd25e9d6 100644 --- a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -2,21 +2,30 @@ import java.util.ArrayList; import java.util.List; +import javax.annotation.Nullable; +import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBBugs; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; public class QuestDBTableGenerator { - public SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + public SQLQueryAdapter getQuery(QuestDBGlobalState globalState, @Nullable String tableName) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder(); - String tableName = "test"; // globalState.getSchema().getFreeTableName(); + String name = tableName; + if (tableName == null) { + name = globalState.getSchema().getFreeTableName(); + } sb.append("CREATE TABLE "); - sb.append(tableName); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(name); sb.append("("); List columns = getNewColumns(); for (int i = 0; i < columns.size(); i++) { @@ -27,13 +36,27 @@ public SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { sb.append(" "); sb.append(columns.get(i).getType()); } - sb.append(");"); + sb.append(")"); + // test index at Create Table + if (Randomly.getBooleanWithRatherLowProbability()) { + errors.add("cannot create index"); + // QuestDB does not support index for non-SYMBOL typed columns + errors.add("Index flag is only supported for SYMBOL"); + sb.append(","); + String index = String.format(" INDEX (%s)", Randomly.fromList(columns).getName()); + sb.append(index); + } + if (QuestDBBugs.bug2689) { + errors.add("Invalid metadata"); + } + sb.append(";"); + errors.add("table already exists"); return new SQLQueryAdapter(sb.toString(), errors, true); } private static List getNewColumns() { List columns = new ArrayList<>(); - for (int i = 0; i < sqlancer.Randomly.smallNumber() + 1; i++) { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { String columnName = String.format("c%d", i); QuestDBCompositeDataType columnType = QuestDBCompositeDataType.getRandomWithoutNull(); columns.add(new QuestDBColumn(columnName, columnType, false)); diff --git a/src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java b/src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java new file mode 100644 index 000000000..19c48f9f7 --- /dev/null +++ b/src/sqlancer/questdb/gen/QuestDBTruncateGenerator.java @@ -0,0 +1,22 @@ +package sqlancer.questdb.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; + +public final class QuestDBTruncateGenerator { + private QuestDBTruncateGenerator() { + + } + + public static SQLQueryAdapter generate(QuestDBGlobalState globalState) { + StringBuilder sb = new StringBuilder("TRUNCATE TABLE "); + ExpectedErrors errors = new ExpectedErrors(); + QuestDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + QuestDBErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java new file mode 100644 index 000000000..fb6bd3c3f --- /dev/null +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -0,0 +1,80 @@ +package sqlancer.questdb.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBTable; +import sqlancer.questdb.QuestDBSchema.QuestDBTables; +import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.questdb.ast.QuestDBSelect; +import sqlancer.questdb.gen.QuestDBExpressionGenerator; + +public class QuestDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, QuestDBGlobalState> implements TestOracle { + + QuestDBSchema s; + QuestDBTables targetTables; + QuestDBExpressionGenerator gen; + QuestDBSelect select; + + protected QuestDBQueryPartitioningBase(QuestDBGlobalState state) { + super(state); + QuestDBErrors.addExpressionErrors(errors); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new QuestDBColumn("*", null, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + } + return columns; + } + + public static String canonicalizeResultValue(String value) { + // Rule: -0.0 should be canonicalized to 0.0 + if (Objects.equals(value, "-0.0")) { + return "0.0"; + } + + return value; + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new QuestDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new QuestDBSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + // Ignore JOINs for now + select.setFromList(new ArrayList<>(tableList)); + select.setWhereClause(null); + } +} diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java new file mode 100644 index 000000000..fe5b5890d --- /dev/null +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java @@ -0,0 +1,41 @@ +package sqlancer.questdb.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.questdb.QuestDBErrors; +import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBToStringVisitor; + +public class QuestDBQueryPartitioningWhereTester extends QuestDBQueryPartitioningBase { + public QuestDBQueryPartitioningWhereTester(QuestDBGlobalState state) { + super(state); + QuestDBErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = QuestDBToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + // Ignore OrderBy for now + + select.setWhereClause(predicate); + String firstQueryString = QuestDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = QuestDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = QuestDBToStringVisitor.asString(select); + + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, false, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, QuestDBQueryPartitioningBase::canonicalizeResultValue); + } +} From 488ca7d586fa3ba2c6e94216b902205f5ca5df1b Mon Sep 17 00:00:00 2001 From: Suri Zhang Date: Thu, 3 Nov 2022 15:22:33 +0800 Subject: [PATCH 234/856] Implement more expressions and add SELECT, WHERE to test --- src/sqlancer/questdb/QuestDBBugs.java | 3 - src/sqlancer/questdb/QuestDBErrors.java | 7 +++ .../questdb/QuestDBToStringVisitor.java | 41 ++++++++++++- src/sqlancer/questdb/ast/QuestDBConstant.java | 4 +- .../gen/QuestDBExpressionGenerator.java | 58 ++++++++++++++++++- .../questdb/gen/QuestDBTableGenerator.java | 13 ----- 6 files changed, 105 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/questdb/QuestDBBugs.java b/src/sqlancer/questdb/QuestDBBugs.java index bbba765a3..1bd565823 100644 --- a/src/sqlancer/questdb/QuestDBBugs.java +++ b/src/sqlancer/questdb/QuestDBBugs.java @@ -2,9 +2,6 @@ public final class QuestDBBugs { - // https://github.com/questdb/questdb/issues/2689 - public static boolean bug2689 = true; - private QuestDBBugs() { } } diff --git a/src/sqlancer/questdb/QuestDBErrors.java b/src/sqlancer/questdb/QuestDBErrors.java index 44dbea148..83c74fdc1 100644 --- a/src/sqlancer/questdb/QuestDBErrors.java +++ b/src/sqlancer/questdb/QuestDBErrors.java @@ -9,6 +9,13 @@ private QuestDBErrors() { public static void addExpressionErrors(ExpectedErrors errors) { // TODO (anxing) + errors.add("unexpected argument for function: "); + errors.add("unexpected token:"); // SELECT FROM multiple tables without WHERE/ JOIN clause + errors.add("boolean expression expected"); + errors.add("Column name expected"); + errors.add("too few arguments for 'in'"); + errors.add("cannot compare TIMESTAMP with type"); // WHERE column IN with nonTIMESTAMP arg + errors.add("constant expected"); } public static void addGroupByErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/questdb/QuestDBToStringVisitor.java b/src/sqlancer/questdb/QuestDBToStringVisitor.java index 7552b06cb..295a6d910 100644 --- a/src/sqlancer/questdb/QuestDBToStringVisitor.java +++ b/src/sqlancer/questdb/QuestDBToStringVisitor.java @@ -4,6 +4,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.questdb.ast.QuestDBConstant; import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.questdb.ast.QuestDBSelect; public class QuestDBToStringVisitor extends NewToStringVisitor { @@ -11,7 +12,9 @@ public class QuestDBToStringVisitor extends NewToStringVisitor expr) { if (expr instanceof QuestDBConstant) { visit((QuestDBConstant) expr); - } else { // TODO: maybe implement QuestDBSelect & Join + } else if (expr instanceof QuestDBSelect) { + visit((QuestDBSelect) expr); + } else { // TODO: maybe implement QuestDBJoin throw new AssertionError("Unknown class: " + expr.getClass()); } } @@ -20,6 +23,42 @@ private void visit(QuestDBConstant constant) { sb.append(constant.toString()); } + private void visit(QuestDBSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + // if (!select.getGroupByExpressions().isEmpty()) { + // sb.append(" GROUP BY "); + // visit(select.getGroupByExpressions()); + // } + // if (select.getHavingClause() != null) { + // sb.append(" HAVING "); + // visit(select.getHavingClause()); + // } + // if (!select.getOrderByExpressions().isEmpty()) { + // sb.append(" ORDER BY "); + // visit(select.getOrderByExpressions()); + // } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + } + public static String asString(Node expr) { QuestDBToStringVisitor visitor = new QuestDBToStringVisitor(); visitor.visit(expr); diff --git a/src/sqlancer/questdb/ast/QuestDBConstant.java b/src/sqlancer/questdb/ast/QuestDBConstant.java index e0950e656..a59ce1053 100644 --- a/src/sqlancer/questdb/ast/QuestDBConstant.java +++ b/src/sqlancer/questdb/ast/QuestDBConstant.java @@ -66,9 +66,9 @@ public double getValue() { @Override public String toString() { if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; + return "cast('Infinity' as double)"; } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; + return "cast('-Infinity' as double)"; } return String.valueOf(value); } diff --git a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java index 82f82ac88..8f9ea443a 100644 --- a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java @@ -1,8 +1,14 @@ package sqlancer.questdb.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewInOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; @@ -21,6 +27,10 @@ public QuestDBExpressionGenerator(QuestDBGlobalState globalState) { this.globalState = globalState; } + private enum Expression { + UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, IN + } + @Override public Node negatePredicate(Node predicate) { return new NewUnaryPrefixOperatorNode<>(predicate, QuestDBUnaryPrefixOperator.NOT); @@ -57,7 +67,31 @@ public Node generateConstant() { protected Node generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); - } else { + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + + switch (expr) { + case UNARY_PREFIX: + return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), + QuestDBUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), + QuestDBUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + QuestDBBinaryComparisonOperator.getRandom()); + case BINARY_ARITHMETIC: + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + QuestDBBinaryArithmeticOperator.getRandom()); + case BINARY_LOGICAL: + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + QuestDBBinaryLogicalOperator.getRandom()); + case IN: + return new NewInOperatorNode<>(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + default: throw new AssertionError("Expression generation failed, depth=" + depth); } } @@ -125,7 +159,7 @@ public static Operator getRandom() { public enum QuestDBBinaryComparisonOperator implements Operator { EQUALS("="), GREATER_THAN(">"), GREATER_THAN_EQUALS(">="), LESS_THAN("<"), SMALLER_THAN_EQUALS("<="), - NOT_EQUALS("!="); + NOT_EQUALS("!="), REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); private String textRepr; @@ -143,4 +177,24 @@ public String getTextRepresentation() { } } + + public enum QuestDBBinaryArithmeticOperator implements Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"); // , LSHIFT("<<"), + // RSHIFT(">>"); + + private String textRepr; + + QuestDBBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } } diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java index abd25e9d6..a308b8d10 100644 --- a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -7,7 +7,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.questdb.QuestDBBugs; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; @@ -37,18 +36,6 @@ public SQLQueryAdapter getQuery(QuestDBGlobalState globalState, @Nullable String sb.append(columns.get(i).getType()); } sb.append(")"); - // test index at Create Table - if (Randomly.getBooleanWithRatherLowProbability()) { - errors.add("cannot create index"); - // QuestDB does not support index for non-SYMBOL typed columns - errors.add("Index flag is only supported for SYMBOL"); - sb.append(","); - String index = String.format(" INDEX (%s)", Randomly.fromList(columns).getName()); - sb.append(index); - } - if (QuestDBBugs.bug2689) { - errors.add("Invalid metadata"); - } sb.append(";"); errors.add("table already exists"); return new SQLQueryAdapter(sb.toString(), errors, true); From 201f2751c13e1cfe2aff99f18a21496b8e71e37e Mon Sep 17 00:00:00 2001 From: "Anxing (Suri) Zhang" Date: Fri, 4 Nov 2022 12:33:35 +0800 Subject: [PATCH 235/856] Update README.md Add QuestDB to supported DBMS --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index eb0fcd779..af793f42c 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | OceanBase | Working | Untyped | | | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | | Databend | Working | Typed | | +| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | + # Using SQLancer From 913d53bceca42e576e010b035460b83267640119 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Fri, 4 Nov 2022 14:42:29 +0100 Subject: [PATCH 236/856] Use new ClickHouse DockerHub Updated clickhouse-jdbc, added functions support and many improvements in column and tables names --- .github/workflows/main.yml | 6 +- src/sqlancer/clickhouse/ClickHouseErrors.java | 9 + src/sqlancer/clickhouse/ClickHouseSchema.java | 20 +- .../clickhouse/ClickHouseToStringVisitor.java | 95 +- .../clickhouse/ClickHouseVisitor.java | 9 +- .../clickhouse/ast/ClickHouseAggregate.java | 3 +- .../ClickHouseBinaryArithmeticOperation.java | 65 + .../ClickHouseBinaryComparisonOperation.java | 18 +- .../ClickHouseBinaryFunctionOperation.java | 57 + .../ast/ClickHouseBinaryLogicalOperation.java | 23 +- .../clickhouse/ast/ClickHouseCast.java | 42 +- .../ast/ClickHouseCastOperation.java | 3 +- .../ast/ClickHouseColumnReference.java | 19 +- .../clickhouse/ast/ClickHouseConstant.java | 1894 +---------------- .../clickhouse/ast/ClickHouseExpression.java | 50 +- .../clickhouse/ast/ClickHouseSelect.java | 18 +- .../ast/ClickHouseTableReference.java | 19 +- .../ast/ClickHouseUnaryFunctionOperation.java | 52 + .../ast/ClickHouseUnaryPostfixOperation.java | 3 +- .../ast/ClickHouseUnaryPrefixOperation.java | 12 +- .../constant/ClickHouseBooleanConstant.java | 142 ++ .../constant/ClickHouseCreateConstant.java | 162 ++ .../constant/ClickHouseFloat32Constant.java | 136 ++ .../constant/ClickHouseFloat64Constant.java | 136 ++ .../constant/ClickHouseInt128Constant.java | 139 ++ .../ast/constant/ClickHouseInt16Constant.java | 130 ++ .../constant/ClickHouseInt256Constant.java | 139 ++ .../ast/constant/ClickHouseInt32Constant.java | 130 ++ .../ast/constant/ClickHouseInt64Constant.java | 131 ++ .../ast/constant/ClickHouseInt8Constant.java | 130 ++ .../ast/constant/ClickHouseNullConstant.java | 53 + .../constant/ClickHouseStringConstant.java | 125 ++ .../constant/ClickHouseUInt128Constant.java | 139 ++ .../constant/ClickHouseUInt16Constant.java | 130 ++ .../constant/ClickHouseUInt256Constant.java | 139 ++ .../constant/ClickHouseUInt32Constant.java | 130 ++ .../constant/ClickHouseUInt64Constant.java | 131 ++ .../ast/constant/ClickHouseUInt8Constant.java | 138 ++ .../clickhouse/gen/ClickHouseCommon.java | 2 +- .../gen/ClickHouseExpressionGenerator.java | 132 +- .../gen/ClickHouseTableGenerator.java | 16 +- .../oracle/norec/ClickHouseNoRECOracle.java | 29 +- .../tlp/ClickHouseTLPAggregateOracle.java | 60 +- .../oracle/tlp/ClickHouseTLPBase.java | 44 +- .../tlp/ClickHouseTLPGroupByOracle.java | 13 - .../oracle/tlp/ClickHouseTLPHavingOracle.java | 33 +- ...ickHouseBinaryComparisonOperationTest.java | 171 +- .../ast/ClickHouseOperatorsVisitorTest.java | 117 + .../ast/ClickHouseToStringVisitorTest.java | 345 +++ 49 files changed, 3535 insertions(+), 2204 deletions(-) create mode 100644 src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java create mode 100644 src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java create mode 100644 src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java create mode 100644 src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java create mode 100644 test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java create mode 100644 test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 348bdd976..67ade8e58 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -87,11 +87,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up ClickHouse run: | - docker pull yandex/clickhouse-server:latest - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d yandex/clickhouse-server:latest + docker pull clickhouse/clickhouse-server:latest + docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:latest sleep 5 - name: Run Tests - run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest test + run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test cockroachdb: diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 88597d681..d2b6c700a 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -54,6 +54,15 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("there are only columns"); errors.add("there are columns"); errors.add("in block. (NOT_FOUND_COLUMN_IN_BLOCK)"); + errors.add("Missing columns"); + errors.add("Ambiguous column"); + errors.add("Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)"); + errors.add("Floating point partition key is not supported"); + errors.add("Cannot get JOIN keys from JOIN ON section"); + errors.add("ILLEGAL_DIVISION"); + errors.add("DECIMAL_OVERFLOW"); + errors.add("Cannot convert out of range floating point value to integer type"); + errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/42768 } } diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index ff58fc7a9..97f4534a1 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -8,12 +8,15 @@ import java.util.List; import java.util.Map; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseConstant; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; @@ -34,7 +37,7 @@ public ClickHouseLancerDataType(ClickHouseDataType type) { } public ClickHouseLancerDataType(String textRepr) { - this.clickHouseType = ClickHouseDataType.fromTypeString(textRepr); + this.clickHouseType = ClickHouseDataType.of(textRepr); this.textRepr = textRepr; } @@ -78,6 +81,11 @@ public boolean isAlias() { public boolean isMaterialized() { return isMaterialized; } + + public ClickHouseColumnReference asColumnReference(String tableAlias) { + return new ClickHouseColumnReference(this, null, tableAlias); + } + } public static ClickHouseConstant getConstant(ResultSet randomRowValues, int columnIndex, @@ -85,20 +93,20 @@ public static ClickHouseConstant getConstant(ResultSet randomRowValues, int colu Object value; ClickHouseConstant constant; if (randomRowValues.getString(columnIndex) == null) { - constant = ClickHouseConstant.createNullConstant(); + constant = ClickHouseCreateConstant.createNullConstant(); } else { switch (valueType) { case Int32: value = randomRowValues.getLong(columnIndex); - constant = ClickHouseConstant.createInt32Constant((long) value); + constant = ClickHouseCreateConstant.createInt32Constant((long) value); break; case Float64: value = randomRowValues.getDouble(columnIndex); - constant = ClickHouseConstant.createFloat64Constant((double) value); + constant = ClickHouseCreateConstant.createFloat64Constant((double) value); break; case String: value = randomRowValues.getString(columnIndex); - constant = ClickHouseConstant.createStringConstant((String) value); + constant = ClickHouseCreateConstant.createStringConstant((String) value); break; case AggregateFunction: case Array: diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index 2d78fcc8c..cbc622560 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -1,8 +1,10 @@ package sqlancer.clickhouse; +import java.util.List; + import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; -import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; @@ -21,17 +23,6 @@ public void visitSpecific(ClickHouseExpression expr) { ClickHouseVisitor.super.visit(expr); } - @Override - public void visit(ClickHouseBinaryComparisonOperation op) { - sb.append("("); - visit(op.getLeft()); - sb.append(") "); - sb.append(op.getOperator().getTextRepresentation()); - sb.append(" ("); - visit(op.getRight()); - sb.append(")"); - } - @Override public void visit(ClickHouseBinaryLogicalOperation op) { sb.append("("); @@ -83,8 +74,17 @@ public void visit(ClickHouseSelect select, boolean inner) { } visit(select.getFetchColumns()); - sb.append(" FROM "); - visit(select.getFromList()); + ClickHouseExpression fromClause = select.getFromClause(); + if (fromClause != null) { + sb.append(" FROM "); + visit(fromClause); + } + List joins = select.getJoinClauses(); + if (!joins.isEmpty()) { + for (ClickHouseExpression.ClickHouseJoin join : joins) { + visit(join); + } + } if (select.getWhereClause() != null) { sb.append(" WHERE "); visit(select.getWhereClause()); @@ -108,7 +108,12 @@ public void visit(ClickHouseSelect select, boolean inner) { @Override public void visit(ClickHouseTableReference tableReference) { - sb.append(tableReference.getTable().getName()); + sb.append(tableReference.getTable().getName()); // Original name, not alias. + String alias = tableReference.getAlias(); + if (alias != null) { + sb.append(" AS " + alias); + } + } @Override @@ -130,24 +135,76 @@ public void visit(ClickHouseCastOperation cast) { @Override public void visit(ClickHouseExpression.ClickHouseJoin join) { - sb.append("ON "); - visit(join.getOnClause()); + ClickHouseExpression.ClickHouseJoin.JoinType type = join.getType(); + if (type == ClickHouseExpression.ClickHouseJoin.JoinType.CROSS) { + sb.append(" JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.INNER) { + sb.append(" INNER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_OUTER) { + sb.append(" LEFT OUTER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_OUTER) { + sb.append(" RIGHT OUTER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.FULL_OUTER) { + sb.append(" FULL OUTER JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_SEMI) { + sb.append(" LEFT SEMI JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_SEMI) { + sb.append(" RIGHT SEMI JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_ANTI) { + sb.append(" LEFT ANTI JOIN "); + visit(join.getRightTable()); + } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_ANTI) { + sb.append(" RIGHT ANTI JOIN "); + visit(join.getRightTable()); + } else { + throw new UnsupportedOperationException(); + } + ClickHouseExpression onClause = join.getOnClause(); + if (onClause != null) { + sb.append(" ON "); + visit(onClause); + } } @Override public void visit(ClickHouseColumnReference c) { - if (c.getColumn().getTable() == null) { + if (c.getTableAlias() != null) { + sb.append(c.getTableAlias()); + sb.append("."); + sb.append(c.getColumn().getName()); + } else if (c.getColumn().getTable() == null) { sb.append(c.getColumn().getName()); } else { sb.append(c.getColumn().getFullQualifiedName()); } + if (c.getAlias() != null) { + sb.append(" AS " + c.getAlias()); + } + } + + @Override + public void visit(ClickHouseBinaryFunctionOperation func) { + sb.append(func.getOperatorRepresentation()); + sb.append("("); + visit(func.getLeft()); + sb.append(","); + visit(func.getRight()); + sb.append(")"); } @Override public void visit(ClickHouseAliasOperation alias) { visit(alias.getExpression()); - sb.append(" AS "); + sb.append(" AS `"); sb.append(alias.getAlias()); + sb.append("`"); } public static String asString(ClickHouseExpression expr) { diff --git a/src/sqlancer/clickhouse/ClickHouseVisitor.java b/src/sqlancer/clickhouse/ClickHouseVisitor.java index 93a46d607..2966f93be 100644 --- a/src/sqlancer/clickhouse/ClickHouseVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseVisitor.java @@ -3,6 +3,7 @@ import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseCastOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; @@ -58,8 +59,12 @@ default void visit(ClickHouseExpression.ClickHousePostfixText op) { void visit(ClickHouseAggregate aggregate); + void visit(ClickHouseBinaryFunctionOperation func); + default void visit(ClickHouseExpression expr) { - if (expr instanceof ClickHouseBinaryComparisonOperation) { + if (expr instanceof ClickHouseBinaryFunctionOperation) { + visit((ClickHouseBinaryFunctionOperation) expr); + } else if (expr instanceof ClickHouseBinaryComparisonOperation) { visit((ClickHouseBinaryComparisonOperation) expr); } else if (expr instanceof ClickHouseBinaryLogicalOperation) { visit((ClickHouseBinaryLogicalOperation) expr); @@ -83,6 +88,8 @@ default void visit(ClickHouseExpression expr) { visit((ClickHouseAggregate) expr); } else if (expr instanceof ClickHouseAliasOperation) { visit((ClickHouseAliasOperation) expr); + } else if (expr instanceof ClickHouseExpression.ClickHouseJoinOnClause) { + visit((ClickHouseExpression.ClickHouseJoinOnClause) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java index 172554b77..e868f82ca 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java @@ -4,7 +4,8 @@ import java.util.List; import java.util.stream.Collectors; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseSchema; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java new file mode 100644 index 000000000..fc0f8504a --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryArithmeticOperation.java @@ -0,0 +1,65 @@ +package sqlancer.clickhouse.ast; + +import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; + +public class ClickHouseBinaryArithmeticOperation extends ClickHouseExpression + implements BinaryOperation { + + public enum ClickHouseBinaryArithmeticOperator { + ADD("+"), // + MINUS("-"), // + MULT("*"), // + DIV("/"), // + MODULO("%"); // + + String textRepresentation; + + ClickHouseBinaryArithmeticOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static ClickHouseBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + + private final ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator operation; + private final ClickHouseExpression left; + private final ClickHouseExpression right; + + public ClickHouseBinaryArithmeticOperation(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator operation) { + this.left = left; + this.right = right; + this.operation = operation; + } + + public ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator getOperator() { + return operation; + } + + @Override + public ClickHouseExpression getLeft() { + return left; + } + + @Override + public ClickHouseExpression getRight() { + return right; + } + + @Override + public String getOperatorRepresentation() { + return operation.getTextRepresentation(); + } + + public static ClickHouseBinaryArithmeticOperation create(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator op) { + return new ClickHouseBinaryArithmeticOperation(left, right, op); + } +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java index 6696a3ee3..75dc259e8 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperation.java @@ -1,8 +1,10 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.visitor.BinaryOperation; public class ClickHouseBinaryComparisonOperation extends ClickHouseExpression @@ -75,7 +77,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { } else if (lessThan.asInt() >= 1) { return lessThan; } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } } @@ -95,7 +97,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { && equals.getDataType() == ClickHouseDataType.UInt32 && equals.getDataType() == ClickHouseDataType.Int64 && equals.getDataType() == ClickHouseDataType.UInt64 && equals.asInt() == 1) { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } else { ClickHouseConstant applyLess = left.applyLess(right); if (applyLess == null) { @@ -121,7 +123,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { && lessThan.getDataType() == ClickHouseDataType.UInt32 && lessThan.getDataType() == ClickHouseDataType.Int64 && lessThan.getDataType() == ClickHouseDataType.UInt64 && lessThan.asInt() >= 1) { - return ClickHouseConstant.createTrue(); + return ClickHouseCreateConstant.createTrue(); } else { ClickHouseConstant applyLess = left.applyLess(right); if (applyLess == null) { @@ -146,14 +148,14 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { return null; } if (left.isNull() || right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { ClickHouseConstant applyEquals = left.applyEquals(right); if (applyEquals == null) { return null; } boolean equals = applyEquals.asInt() == 1; - return ClickHouseConstant.createBoolean(!equals); + return ClickHouseCreateConstant.createBoolean(!equals); } } @@ -165,7 +167,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { return null; } if (left.isNull() || right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } ClickHouseConstant leftStr = ClickHouseCast.castToText(left); ClickHouseConstant rightStr = ClickHouseCast.castToText(right); @@ -173,7 +175,7 @@ ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { return null; } boolean val = LikeImplementationHelper.match(leftStr.asString(), rightStr.asString(), 0, 0, false); - return ClickHouseConstant.createBoolean(val); + return ClickHouseCreateConstant.createBoolean(val); } }; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java new file mode 100644 index 000000000..65bf4c79e --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryFunctionOperation.java @@ -0,0 +1,57 @@ +package sqlancer.clickhouse.ast; + +import sqlancer.Randomly; + +public class ClickHouseBinaryFunctionOperation extends ClickHouseExpression { + + public enum ClickHouseBinaryFunctionOperator { + INT_DIV("intDiv"), GCD("gcd"), LCM("lcm"), MAX2("max2"), MIN2("min2"), POW("pow"); + + String textRepresentation; + + ClickHouseBinaryFunctionOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static ClickHouseBinaryFunctionOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public String getTextRepresentation() { + return textRepresentation; + } + } + + private final ClickHouseBinaryFunctionOperator operation; + private final ClickHouseExpression left; + private final ClickHouseExpression right; + + public ClickHouseBinaryFunctionOperation(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryFunctionOperator operation) { + this.left = left; + this.right = right; + this.operation = operation; + } + + public ClickHouseBinaryFunctionOperator getOperator() { + return operation; + } + + public ClickHouseExpression getLeft() { + return left; + } + + public ClickHouseExpression getRight() { + return right; + } + + public String getOperatorRepresentation() { + return operation.getTextRepresentation(); + } + + public static ClickHouseBinaryFunctionOperation create(ClickHouseExpression left, ClickHouseExpression right, + ClickHouseBinaryFunctionOperator op) { + return new ClickHouseBinaryFunctionOperation(left, right, op); + } + +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java index bbd37e865..019fad752 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseBinaryLogicalOperation.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.ast; import sqlancer.Randomly; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; public class ClickHouseBinaryLogicalOperation extends ClickHouseExpression { @@ -14,22 +15,22 @@ public enum ClickHouseBinaryLogicalOperator { @Override public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { if (left.isNull() && right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else if (left.isNull()) { if (right.asBooleanNotNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } else if (right.isNull()) { if (left.asBooleanNotNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } else { - return left.asBooleanNotNull() && right.asBooleanNotNull() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); + return left.asBooleanNotNull() && right.asBooleanNotNull() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); } } }, @@ -37,13 +38,13 @@ public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant righ @Override public ClickHouseConstant apply(ClickHouseConstant left, ClickHouseConstant right) { if (!left.isNull() && left.asBooleanNotNull()) { - return ClickHouseConstant.createTrue(); + return ClickHouseCreateConstant.createTrue(); } else if (!right.isNull() && right.asBooleanNotNull()) { - return ClickHouseConstant.createTrue(); + return ClickHouseCreateConstant.createTrue(); } else if (left.isNull() || right.isNull()) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return ClickHouseConstant.createFalse(); + return ClickHouseCreateConstant.createFalse(); } } }; diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCast.java b/src/sqlancer/clickhouse/ast/ClickHouseCast.java index dc35f1fc7..09c946717 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCast.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCast.java @@ -6,7 +6,9 @@ import java.util.Optional; import java.util.regex.Pattern; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; public final class ClickHouseCast extends ClickHouseExpression { @@ -50,18 +52,18 @@ public static Optional isTrue(ClickHouseConstant value) { public static ClickHouseConstant castToInt(ClickHouseConstant cons) { switch (cons.getDataType()) { case Nothing: - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); case Int32: return cons; case Float64: - return ClickHouseConstant.createInt32Constant((long) cons.asDouble()); + return ClickHouseCreateConstant.createInt32Constant((long) cons.asDouble()); case String: String asString = cons.asString(); while (startsWithWhitespace(asString)) { asString = asString.substring(1); } if (!asString.isEmpty() && unprintAbleCharThatLetsBecomeNumberZero(asString)) { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } for (int i = asString.length(); i >= 0; i--) { try { @@ -79,13 +81,13 @@ public static ClickHouseConstant castToInt(ClickHouseConstant cons) { result = Long.MAX_VALUE; } } - return ClickHouseConstant.createInt32Constant(result); + return ClickHouseCreateConstant.createInt32Constant(result); } } catch (Exception e) { } } - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); default: throw new AssertionError(); } @@ -95,7 +97,7 @@ public static ClickHouseConstant castToInt(ClickHouseConstant cons) { public static ClickHouseConstant castToReal(ClickHouseConstant cons) { ClickHouseConstant numericValue = castToNumeric(cons); if (numericValue.getDataType() == ClickHouseDataType.Int32) { - return ClickHouseConstant.createFloat64Constant(numericValue.asInt()); + return ClickHouseCreateConstant.createFloat64Constant(numericValue.asInt()); } else { return numericValue; } @@ -120,7 +122,7 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool boolean noNumIsRealZero, boolean convertIntToReal) throws AssertionError { switch (value.getDataType()) { case Nothing: - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); case Int32: case Float64: return value; @@ -130,11 +132,11 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool asString = asString.substring(1); } if (!asString.isEmpty() && unprintAbleCharThatLetsBecomeNumberZero(asString)) { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } if (asString.toLowerCase().startsWith("-infinity") || asString.toLowerCase().startsWith("infinity") || asString.startsWith("NaN")) { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } for (int i = asString.length(); i >= 0; i--) { try { @@ -151,17 +153,17 @@ private static ClickHouseConstant convertInternal(ClickHouseConstant value, bool boolean isInteger = !isFloatingPointNumber && first.compareTo(second) == 0; if (doubleShouldBeConvertedToInt || isInteger && !convertIntToReal) { // see https://www.sqlite.org/src/tktview/afdc5a29dc - return ClickHouseConstant.createInt32Constant(first.longValue()); + return ClickHouseCreateConstant.createInt32Constant(first.longValue()); } else { - return ClickHouseConstant.createFloat64Constant(d); + return ClickHouseCreateConstant.createFloat64Constant(d); } } catch (Exception e) { } } if (noNumIsRealZero) { - return ClickHouseConstant.createFloat64Constant(0.0); + return ClickHouseCreateConstant.createFloat64Constant(0.0); } else { - return ClickHouseConstant.createInt32Constant(0); + return ClickHouseCreateConstant.createInt32Constant(0); } default: throw new AssertionError(value); @@ -222,14 +224,14 @@ public static ClickHouseConstant castToText(ClickHouseConstant cons) { } if (cons.getDataType() == ClickHouseDataType.Float64) { if (cons.asDouble() == Double.POSITIVE_INFINITY) { - return ClickHouseConstant.createStringConstant("Inf"); + return ClickHouseCreateConstant.createStringConstant("Inf"); } else if (cons.asDouble() == Double.NEGATIVE_INFINITY) { - return ClickHouseConstant.createStringConstant("-Inf"); + return ClickHouseCreateConstant.createStringConstant("-Inf"); } return castRealToText(cons); } if (cons.getDataType() == ClickHouseDataType.Int32) { - return ClickHouseConstant.createStringConstant(String.valueOf(cons.asInt())); + return ClickHouseCreateConstant.createStringConstant(String.valueOf(cons.asInt())); } return null; } @@ -237,7 +239,7 @@ public static ClickHouseConstant castToText(ClickHouseConstant cons) { private static synchronized ClickHouseConstant castRealToText(ClickHouseConstant cons) throws AssertionError { try (Statement s = castDatabase.createStatement()) { String castResult = s.executeQuery("SELECT CAST(" + cons.asDouble() + " AS TEXT)").getString(1); - return ClickHouseConstant.createStringConstant(castResult); + return ClickHouseCreateConstant.createStringConstant(castResult); } catch (Exception e) { throw new AssertionError(e); } @@ -246,9 +248,9 @@ private static synchronized ClickHouseConstant castRealToText(ClickHouseConstant public static ClickHouseConstant asBoolean(ClickHouseConstant val) { Optional boolVal = isTrue(val); if (boolVal.isPresent()) { - return ClickHouseConstant.createBoolean(boolVal.get()); + return ClickHouseCreateConstant.createBoolean(boolVal.get()); } else { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java index 490a77d37..a905b4e49 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseCastOperation.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; public class ClickHouseCastOperation extends ClickHouseExpression { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java index 796eb8333..6d75cf5fa 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseColumnReference.java @@ -5,17 +5,30 @@ public class ClickHouseColumnReference extends ClickHouseExpression { private final ClickHouseColumn column; + private final String columnAlias; + private final String tableAlias; - public ClickHouseColumnReference(ClickHouseColumn column) { + public ClickHouseColumnReference(ClickHouseColumn column, String columnAlias, String tableAlias) { this.column = column; + this.columnAlias = columnAlias; + this.tableAlias = tableAlias; } - public static ClickHouseColumnReference create(ClickHouseColumn column) { - return new ClickHouseColumnReference(column); + public ClickHouseColumnReference(ClickHouseAliasOperation alias) { + this.column = new ClickHouseColumn(alias.getAlias(), null, true, false, null); + this.columnAlias = null; + this.tableAlias = null; } public ClickHouseColumn getColumn() { return column; } + public String getAlias() { + return columnAlias; + } + + public String getTableAlias() { + return tableAlias; + } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java index 048c14296..b38ad7e89 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseConstant.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseConstant.java @@ -1,1899 +1,13 @@ package sqlancer.clickhouse.ast; -import java.math.BigInteger; +import com.clickhouse.client.ClickHouseDataType; -import ru.yandex.clickhouse.domain.ClickHouseDataType; -import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; public abstract class ClickHouseConstant extends ClickHouseExpression { - public static class ClickHouseNullConstant extends ClickHouseConstant { - - @Override - public String toString() { - return "NULL"; - } - - @Override - public boolean isNull() { - return true; - } - - @Override - public boolean asBooleanNotNull() { - throw new AssertionError(); - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Nothing; - } - - @Override - public boolean compareInternal(Object value) { - return false; - } - - @Override - public ClickHouseConstant applyEquals(ClickHouseConstant right) { - return ClickHouseConstant.createNullConstant(); - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - return ClickHouseConstant.createNullConstant(); - } - - @Override - public Object getValue() { - return null; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - return null; - } - } - - public static class ClickHouseUInt8Constant extends ClickHouseConstant { - - private final int value; - - public ClickHouseUInt8Constant(int value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt8; - } - - @Override - public boolean compareInternal(Object val) { - return value == (int) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case UInt128: - return ClickHouseConstant.createUInt128Constant(BigInteger.valueOf(value)); - case Int128: - return ClickHouseConstant.createInt128Constant(BigInteger.valueOf(value)); - case UInt256: - return ClickHouseConstant.createUInt256Constant(BigInteger.valueOf(value)); - case Int256: - return ClickHouseConstant.createInt256Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant((double) value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt8Constant extends ClickHouseConstant { - - private final int value; - - public ClickHouseInt8Constant(int value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int8; - } - - @Override - public boolean compareInternal(Object val) { - return value == (int) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt16Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseUInt16Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt16; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt16Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseInt16Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int16; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt32Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseUInt32Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt32; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt32Constant extends ClickHouseConstant { - - private final long value; - - public ClickHouseInt32Constant(long value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int32; - } - - @Override - public boolean compareInternal(Object val) { - return value == (long) val; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(value); - case Int8: - return ClickHouseConstant.createInt8Constant(value); - case UInt16: - return ClickHouseConstant.createUInt16Constant(value); - case Int16: - return ClickHouseConstant.createInt16Constant(value); - case UInt32: - return ClickHouseConstant.createUInt32Constant(value); - case Int32: - return ClickHouseConstant.createInt32Constant(value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt64Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseUInt64Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt64; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt64Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseInt64Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int64; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt128Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseUInt128Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt128; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case UInt128: - return ClickHouseConstant.createUInt128Constant(value); - case Int128: - return ClickHouseConstant.createInt128Constant(value); - case UInt256: - return ClickHouseConstant.createUInt256Constant(value); - case Int256: - return ClickHouseConstant.createInt256Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt128Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseInt128Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int128; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case UInt128: - return ClickHouseConstant.createUInt128Constant(value); - case Int128: - return ClickHouseConstant.createInt128Constant(value); - case UInt256: - return ClickHouseConstant.createUInt256Constant(value); - case Int256: - return ClickHouseConstant.createInt256Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseUInt256Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseUInt256Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.UInt256; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case UInt128: - return ClickHouseConstant.createUInt128Constant(value); - case Int128: - return ClickHouseConstant.createInt128Constant(value); - case UInt256: - return ClickHouseConstant.createUInt256Constant(value); - case Int256: - return ClickHouseConstant.createInt256Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseInt256Constant extends ClickHouseConstant { - - private final BigInteger value; - - public ClickHouseInt256Constant(BigInteger value) { - this.value = value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public boolean asBooleanNotNull() { - return value != BigInteger.ZERO; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Int256; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((BigInteger) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asInt() < right.asInt() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public long asInt() { - return value.longValueExact(); - } - - @Override - public Object getValue() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - long val = value.longValueExact(); - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(val); - case Int8: - return ClickHouseConstant.createInt8Constant(val); - case UInt16: - return ClickHouseConstant.createUInt16Constant(val); - case Int16: - return ClickHouseConstant.createInt16Constant(val); - case UInt32: - return ClickHouseConstant.createUInt32Constant(val); - case Int32: - return ClickHouseConstant.createInt32Constant(val); - case UInt64: - return ClickHouseConstant.createUInt64Constant(value); - case Int64: - return ClickHouseConstant.createInt64Constant(value); - case UInt128: - return ClickHouseConstant.createUInt128Constant(value); - case Int128: - return ClickHouseConstant.createInt128Constant(value); - case UInt256: - return ClickHouseConstant.createUInt256Constant(value); - case Int256: - return ClickHouseConstant.createInt256Constant(value); - case Float32: - return ClickHouseConstant.createFloat32Constant(value.floatValue()); - case Float64: - return ClickHouseConstant.createFloat64Constant(value.doubleValue()); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseFloat32Constant extends ClickHouseConstant { - - private final float value; - - public ClickHouseFloat32Constant(float value) { - this.value = value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public String toString() { - if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; - } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; - } - return String.valueOf(value); - } - - @Override - public boolean compareInternal(Object val) { - return Float.compare(value, (float) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asDouble() < right.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - ClickHouseConstant converted = right.cast(ClickHouseDataType.Float32); - return this.asDouble() < converted.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - - @Override - public boolean asBooleanNotNull() { - return Float.compare(value, (float) 0) == 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Float32; - } - - @Override - public double asDouble() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant((long) value); - case Int8: - return ClickHouseConstant.createInt8Constant((long) value); - case UInt16: - return ClickHouseConstant.createUInt16Constant((long) value); - case Int16: - return ClickHouseConstant.createInt16Constant((long) value); - case UInt32: - return ClickHouseConstant.createUInt32Constant((long) value); - case Int32: - return ClickHouseConstant.createInt32Constant((long) value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf((long) value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf((long) value)); - case Float32: - return ClickHouseConstant.createFloat32Constant(value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseFloat64Constant extends ClickHouseConstant { - - private final double value; - - public ClickHouseFloat64Constant(double value) { - this.value = value; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public String toString() { - if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; - } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; - } - return String.valueOf(value); - } - - @Override - public boolean compareInternal(Object val) { - return Double.compare(value, (double) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asDouble() < right.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - ClickHouseConstant converted = right.cast(ClickHouseDataType.Float64); - return this.asDouble() < converted.asDouble() ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - - @Override - public boolean asBooleanNotNull() { - return Double.compare(value, 0.0) == 0; - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.Float64; - } - - @Override - public double asDouble() { - return value; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant((long) value); - case Int8: - return ClickHouseConstant.createInt8Constant((long) value); - case UInt16: - return ClickHouseConstant.createUInt16Constant((long) value); - case Int16: - return ClickHouseConstant.createInt16Constant((long) value); - case UInt32: - return ClickHouseConstant.createUInt32Constant((long) value); - case Int32: - return ClickHouseConstant.createInt32Constant((long) value); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf((long) value)); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf((long) value)); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) value); - case Float64: - return ClickHouseConstant.createFloat64Constant(value); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static class ClickHouseStringConstant extends ClickHouseConstant { - - private final String value; - - public ClickHouseStringConstant(String value) { - this.value = value; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - public Object getValue() { - return value; - } - - @Override - public String toString() { - return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; - } - - @Override - public boolean asBooleanNotNull() { - return value.length() > 0; - } - - @Override - public boolean compareInternal(Object val) { - return value.compareTo((String) val) == 0; - } - - @Override - public ClickHouseConstant applyLess(ClickHouseConstant right) { - if (this.getDataType() == right.getDataType()) { - return this.asString().compareTo(right.asString()) <= 0 ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); - } - throw new IgnoreMeException(); - } - - @Override - public ClickHouseDataType getDataType() { - return ClickHouseDataType.String; - } - - @Override - public ClickHouseConstant cast(ClickHouseDataType type) { - switch (type) { - case String: - return ClickHouseConstant.createStringConstant(this.toString()); - case UInt8: - return ClickHouseConstant.createUInt8Constant(Integer.parseInt(value)); - case Int8: - return ClickHouseConstant.createInt8Constant(Integer.parseInt(value)); - case UInt16: - return ClickHouseConstant.createUInt16Constant(Integer.parseInt(value)); - case Int16: - return ClickHouseConstant.createInt16Constant(Integer.parseInt(value)); - case UInt32: - return ClickHouseConstant.createUInt32Constant(Integer.parseInt(value)); - case Int32: - return ClickHouseConstant.createInt32Constant(Integer.parseInt(value)); - case UInt64: - return ClickHouseConstant.createUInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); - case Int64: - return ClickHouseConstant.createInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); - case Float32: - return ClickHouseConstant.createFloat32Constant((float) Float.parseFloat(value)); - case Float64: - return ClickHouseConstant.createFloat64Constant((double) Double.parseDouble(value)); - case Nothing: - return ClickHouseConstant.createNullConstant(); - case IntervalYear: - case IntervalQuarter: - case IntervalMonth: - case IntervalWeek: - case IntervalDay: - case IntervalHour: - case IntervalMinute: - case IntervalSecond: - case Date: - case DateTime: - case Enum8: - case Enum16: - case Decimal32: - case Decimal64: - case Decimal128: - case Decimal: - case UUID: - case FixedString: - case Nested: - case Tuple: - case Array: - case AggregateFunction: - case Unknown: - default: - throw new AssertionError(type); - } - } - } - - public static ClickHouseConstant createStringConstant(String text) { - return new ClickHouseStringConstant(text); - } - - public static ClickHouseConstant createFloat64Constant(double val) { - return new ClickHouseFloat64Constant(val); - } - - public static ClickHouseConstant createFloat32Constant(float val) { - return new ClickHouseFloat32Constant(val); - } - - public static ClickHouseConstant createIntConstant(ClickHouseDataType type, long val) { - switch (type) { - case IntervalYear: - break; - case IntervalQuarter: - break; - case IntervalMonth: - break; - case IntervalWeek: - break; - case IntervalDay: - break; - case IntervalHour: - break; - case IntervalMinute: - break; - case IntervalSecond: - break; - case UInt256: - return createUInt256Constant(BigInteger.valueOf(val)); - case UInt128: - return createUInt128Constant(BigInteger.valueOf(val)); - case UInt64: - return createUInt64Constant(BigInteger.valueOf(val)); - case UInt32: - return createUInt32Constant(val); - case UInt16: - return createUInt16Constant(val); - case UInt8: - return createUInt8Constant(val); - case Int256: - return createInt256Constant(BigInteger.valueOf(val)); - case Int128: - return createInt256Constant(BigInteger.valueOf(val)); - case Int64: - return createInt64Constant(BigInteger.valueOf(val)); - case Int32: - return createInt32Constant(val); - case Int16: - return createInt16Constant(val); - case Int8: - return createInt8Constant(val); - case Date: - break; - case DateTime: - break; - case Enum8: - break; - case Enum16: - break; - case Float32: - break; - case Float64: - break; - case Decimal32: - break; - case Decimal64: - break; - case Decimal128: - break; - case Decimal: - break; - case UUID: - break; - case String: - break; - case FixedString: - break; - case Nothing: - break; - case Nested: - break; - case Tuple: - break; - case Array: - break; - case AggregateFunction: - break; - case Unknown: - break; - default: - break; - } - throw new AssertionError(type); - } - - public static ClickHouseConstant createInt256Constant(BigInteger val) { - return new ClickHouseInt256Constant(val); - } - - public static ClickHouseConstant createUInt256Constant(BigInteger val) { - return new ClickHouseUInt256Constant(val); - } - - public static ClickHouseConstant createInt128Constant(BigInteger val) { - return new ClickHouseInt128Constant(val); - } - - public static ClickHouseConstant createUInt128Constant(BigInteger val) { - return new ClickHouseUInt128Constant(val); - } - - public static ClickHouseConstant createInt64Constant(BigInteger val) { - return new ClickHouseInt64Constant(val); - } - - public static ClickHouseConstant createUInt64Constant(BigInteger val) { - return new ClickHouseUInt64Constant(val); - } - - public static ClickHouseConstant createInt32Constant(long val) { - return new ClickHouseInt32Constant(val); - } - - public static ClickHouseConstant createUInt32Constant(long val) { - return new ClickHouseUInt32Constant(val); - } - - public static ClickHouseConstant createUInt16Constant(long val) { - return new ClickHouseUInt16Constant(val); - } - - public static ClickHouseConstant createInt16Constant(long val) { - return new ClickHouseInt16Constant(val); - } - - public static ClickHouseConstant createUInt8Constant(long val) { - return new ClickHouseUInt8Constant((int) val); - } - - public static ClickHouseConstant createInt8Constant(long val) { - return new ClickHouseInt8Constant((int) val); - } - public abstract boolean isNull(); - public static ClickHouseConstant createNullConstant() { - return new ClickHouseNullConstant(); - } - - public static ClickHouseConstant createTrue() { - return new ClickHouseUInt8Constant(1); - } - - public static ClickHouseConstant createFalse() { - return new ClickHouseUInt8Constant(0); - } - - public static ClickHouseConstant createBoolean(boolean val) { - return val ? createTrue() : createFalse(); - } - public abstract ClickHouseConstant cast(ClickHouseDataType type); public abstract boolean asBooleanNotNull(); @@ -1904,8 +18,8 @@ public static ClickHouseConstant createBoolean(boolean val) { public ClickHouseConstant applyEquals(ClickHouseConstant right) { if (this.getDataType() == right.getDataType()) { - return this.compareInternal(right.getValue()) ? ClickHouseConstant.createTrue() - : ClickHouseConstant.createFalse(); + return this.compareInternal(right.getValue()) ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); } else { ClickHouseConstant converted = right.cast(this.getDataType()); return this.applyEquals(converted); diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index 161fd0206..34eda72a5 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -1,6 +1,6 @@ package sqlancer.clickhouse.ast; -import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.common.visitor.BinaryOperation; import sqlancer.common.visitor.UnaryOperation; public abstract class ClickHouseExpression { @@ -35,41 +35,67 @@ public ClickHouseExpression getExpression() { } } + public static class ClickHouseJoinOnClause extends ClickHouseExpression + implements BinaryOperation { + private final ClickHouseExpression left; + private final ClickHouseExpression right; + + public ClickHouseJoinOnClause(ClickHouseExpression left, ClickHouseExpression right) { + this.left = left; + this.right = right; + } + + @Override + public final ClickHouseExpression getLeft() { + return this.left; + } + + @Override + public final ClickHouseExpression getRight() { + return this.right; + } + + @Override + public String getOperatorRepresentation() { + return "="; + } + } + public static class ClickHouseJoin extends ClickHouseExpression { // TODO: support ANY, ALL, ASOF modifiers public enum JoinType { - INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, NATURAL, LEFT_SEMI, RIGHT_SEMI, LEFT_ANTI, RIGHT_ANTI; + INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, RIGHT_SEMI, LEFT_ANTI, RIGHT_ANTI; } - private final ClickHouseSchema.ClickHouseTable leftTable; - private final ClickHouseSchema.ClickHouseTable rightTable; - private ClickHouseExpression onClause; + private final ClickHouseTableReference leftTable; + private final ClickHouseTableReference rightTable; + private ClickHouseJoinOnClause onClause; private final ClickHouseJoin.JoinType type; - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable leftTable, ClickHouseSchema.ClickHouseTable rightTable, - ClickHouseJoin.JoinType type, ClickHouseExpression onClause) { + public ClickHouseJoin(ClickHouseTableReference leftTable, ClickHouseTableReference rightTable, + ClickHouseJoin.JoinType type, ClickHouseJoinOnClause onClause) { this.leftTable = leftTable; this.rightTable = rightTable; this.onClause = onClause; this.type = type; } - public ClickHouseJoin(ClickHouseSchema.ClickHouseTable leftTable, ClickHouseSchema.ClickHouseTable rightTable, + public ClickHouseJoin(ClickHouseTableReference leftTable, ClickHouseTableReference rightTable, ClickHouseJoin.JoinType type) { this.leftTable = leftTable; this.rightTable = rightTable; - if (type != ClickHouseJoin.JoinType.NATURAL) { + if (type != ClickHouseJoin.JoinType.CROSS) { throw new AssertionError(); } this.onClause = null; this.type = type; } - public ClickHouseSchema.ClickHouseTable getLeftTable() { + public ClickHouseTableReference getLeftTable() { return leftTable; } - public ClickHouseSchema.ClickHouseTable getTable() { + public ClickHouseTableReference getRightTable() { return rightTable; } @@ -81,7 +107,7 @@ public ClickHouseJoin.JoinType getType() { return type; } - public void setOnClause(ClickHouseExpression onClause) { + public void setOnClause(ClickHouseJoinOnClause onClause) { this.onClause = onClause; } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java index 7a72f2919..24d489982 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java @@ -6,7 +6,7 @@ public class ClickHouseSelect extends ClickHouseExpression { private ClickHouseSelect.SelectType fromOptions = ClickHouseSelect.SelectType.ALL; - private List fromList = Collections.emptyList(); + private ClickHouseExpression fromClause; private ClickHouseExpression whereClause; private List groupByClause = Collections.emptyList(); private ClickHouseExpression limitClause; @@ -24,8 +24,12 @@ public void setSelectType(ClickHouseSelect.SelectType fromOptions) { this.setFromOptions(fromOptions); } - public void setFromTables(List fromTables) { - this.setFromList(fromTables); + public void setFromClause(ClickHouseExpression fromClause) { + this.fromClause = fromClause; + } + + public ClickHouseExpression getFromClause() { + return fromClause; } public ClickHouseSelect.SelectType getFromOptions() { @@ -36,14 +40,6 @@ public void setFromOptions(ClickHouseSelect.SelectType fromOptions) { this.fromOptions = fromOptions; } - public List getFromList() { - return fromList; - } - - public void setFromList(List fromList) { - this.fromList = fromList; - } - public ClickHouseExpression getWhereClause() { return whereClause; } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java b/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java index 415dfca22..406ffb15c 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseTableReference.java @@ -1,17 +1,34 @@ package sqlancer.clickhouse.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; public class ClickHouseTableReference extends ClickHouseExpression { private final ClickHouseTable table; + private final String alias; - public ClickHouseTableReference(ClickHouseTable table) { + public ClickHouseTableReference(ClickHouseTable table, String alias) { this.table = table; + this.alias = alias; } public ClickHouseTable getTable() { return table; } + public String getTableName() { + return (alias == null) ? table.getName() : alias; + } + + public String getAlias() { + return alias; + } + + public List getColumnReferences() { + return this.table.getColumns().stream().map(c -> c.asColumnReference(this.alias)).collect(Collectors.toList()); + } + } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java new file mode 100644 index 000000000..30707354f --- /dev/null +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryFunctionOperation.java @@ -0,0 +1,52 @@ +package sqlancer.clickhouse.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.visitor.UnaryOperation; + +public class ClickHouseUnaryFunctionOperation extends ClickHouseExpression + implements UnaryOperation { + private final ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator operator; + private final ClickHouseExpression expression; + + public ClickHouseUnaryFunctionOperation(ClickHouseExpression expression, ClickHouseUnaryFunctionOperator operator) { + this.operator = operator; + this.expression = expression; + } + + public enum ClickHouseUnaryFunctionOperator implements Operator { + EXP("exp"), SQRT("sqrt"), ERF("erf"), SIN("sin"), COS("cos"), TAN("tan"), SIGN("sign"), RADIANS("radians"), + LOG("log"), ABS("abs"); + + private String textRepresentation; + + ClickHouseUnaryFunctionOperator(String text) { + this.textRepresentation = text; + } + + public static ClickHouseUnaryFunctionOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + + @Override + public ClickHouseExpression getExpression() { + return expression; + } + + @Override + public String getOperatorRepresentation() { + return operator.getTextRepresentation(); + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.PREFIX; + } + +} diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java index 553f85070..444818413 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPostfixOperation.java @@ -1,6 +1,7 @@ package sqlancer.clickhouse.ast; import sqlancer.Randomly; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.visitor.UnaryOperation; @@ -86,7 +87,7 @@ public ClickHouseConstant getExpectedValue() { if (negate) { val = !val; } - return ClickHouseConstant.createInt32Constant(val ? 1 : 0); + return ClickHouseCreateConstant.createInt32Constant(val ? 1 : 0); } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java index 972136e64..15ab8fb36 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseUnaryPrefixOperation.java @@ -1,7 +1,9 @@ package sqlancer.clickhouse.ast; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.visitor.UnaryOperation; @@ -20,10 +22,10 @@ public enum ClickHouseUnaryPrefixOperator implements Operator { @Override public ClickHouseConstant apply(ClickHouseConstant constant) { if (constant.getDataType() == ClickHouseDataType.Nothing) { - return ClickHouseConstant.createNullConstant(); + return ClickHouseCreateConstant.createNullConstant(); } else { - return constant.asBooleanNotNull() ? ClickHouseConstant.createFalse() - : ClickHouseConstant.createTrue(); + return constant.asBooleanNotNull() ? ClickHouseCreateConstant.createFalse() + : ClickHouseCreateConstant.createTrue(); } } }, @@ -31,7 +33,7 @@ public ClickHouseConstant apply(ClickHouseConstant constant) { @Override public ClickHouseConstant apply(ClickHouseConstant constant) { if (constant.getDataType() == ClickHouseDataType.Int32) { - return ClickHouseConstant.createInt32Constant(-constant.asInt()); + return ClickHouseCreateConstant.createInt32Constant(-constant.asInt()); } throw new AssertionError(constant); } diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java new file mode 100644 index 000000000..d5164162c --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseBooleanConstant.java @@ -0,0 +1,142 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseBooleanConstant extends ClickHouseConstant { + + private final boolean value; + + public ClickHouseBooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String toString() { + return value ? "true" : "false"; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Bool; + } + + @Override + public boolean compareInternal(Object val) { + if (val instanceof Boolean) { + return value == ((Boolean) val).booleanValue(); + } else { + return value == ((int) val != 0); + } + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value ? 1 : 0; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(asInt()); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(asInt()); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(asInt()); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(asInt()); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(asInt()); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(asInt()); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(asInt())); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(asInt())); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(BigInteger.valueOf(asInt())); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(BigInteger.valueOf(asInt())); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(BigInteger.valueOf(asInt())); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(BigInteger.valueOf(asInt())); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) asInt()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant((double) asInt()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(asInt() != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java new file mode 100644 index 000000000..dc87619a8 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseCreateConstant.java @@ -0,0 +1,162 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; +import sqlancer.clickhouse.ast.ClickHouseExpression; + +public abstract class ClickHouseCreateConstant extends ClickHouseExpression { + + public static ClickHouseConstant createStringConstant(String text) { + return new ClickHouseStringConstant(text); + } + + public static ClickHouseConstant createFloat64Constant(double val) { + return new ClickHouseFloat64Constant(val); + } + + public static ClickHouseConstant createFloat32Constant(float val) { + return new ClickHouseFloat32Constant(val); + } + + public static ClickHouseConstant createInt256Constant(BigInteger val) { + return new ClickHouseInt256Constant(val); + } + + public static ClickHouseConstant createUInt256Constant(BigInteger val) { + return new ClickHouseUInt256Constant(val); + } + + public static ClickHouseConstant createInt128Constant(BigInteger val) { + return new ClickHouseInt128Constant(val); + } + + public static ClickHouseConstant createUInt128Constant(BigInteger val) { + return new ClickHouseUInt128Constant(val); + } + + public static ClickHouseConstant createInt64Constant(BigInteger val) { + return new ClickHouseInt64Constant(val); + } + + public static ClickHouseConstant createUInt64Constant(BigInteger val) { + return new ClickHouseUInt64Constant(val); + } + + public static ClickHouseConstant createInt32Constant(long val) { + return new ClickHouseInt32Constant(val); + } + + public static ClickHouseConstant createUInt32Constant(long val) { + return new ClickHouseUInt32Constant(val); + } + + public static ClickHouseConstant createUInt16Constant(long val) { + return new ClickHouseUInt16Constant(val); + } + + public static ClickHouseConstant createInt16Constant(long val) { + return new ClickHouseInt16Constant(val); + } + + public static ClickHouseConstant createUInt8Constant(long val) { + return new ClickHouseUInt8Constant((int) val); + } + + public static ClickHouseConstant createInt8Constant(long val) { + return new ClickHouseInt8Constant((int) val); + } + + public static ClickHouseConstant createBooleanConstant(Boolean b) { + return new ClickHouseBooleanConstant(b); + } + + public static ClickHouseConstant createNullConstant() { + return new ClickHouseNullConstant(); + } + + public static ClickHouseConstant createTrue() { + return new ClickHouseBooleanConstant(true); + } + + public static ClickHouseConstant createFalse() { + return new ClickHouseBooleanConstant(false); + } + + public static ClickHouseConstant createBoolean(boolean val) { + return val ? createTrue() : createFalse(); + } + + public static ClickHouseConstant createIntConstant(ClickHouseDataType type, long val) { + switch (type) { + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(BigInteger.valueOf(val)); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(BigInteger.valueOf(val)); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(val)); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(BigInteger.valueOf(val)); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(BigInteger.valueOf(val)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(val)); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + case String: + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } + +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java new file mode 100644 index 000000000..1385790cb --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat32Constant.java @@ -0,0 +1,136 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseFloat32Constant extends ClickHouseConstant { + + private final float value; + + public ClickHouseFloat32Constant(float value) { + this.value = value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + @Override + public boolean compareInternal(Object val) { + return Float.compare(value, (float) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asDouble() < right.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + ClickHouseConstant converted = right.cast(ClickHouseDataType.Float32); + return this.asDouble() < converted.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + + @Override + public boolean asBooleanNotNull() { + return Float.compare(value, (float) 0) == 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Float32; + } + + @Override + public double asDouble() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant((long) value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant((long) value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant((long) value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant((long) value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant((long) value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant((long) value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf((long) value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf((long) value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java new file mode 100644 index 000000000..9146faa39 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseFloat64Constant.java @@ -0,0 +1,136 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseFloat64Constant extends ClickHouseConstant { + + private final double value; + + public ClickHouseFloat64Constant(double value) { + this.value = value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + @Override + public boolean compareInternal(Object val) { + return Double.compare(value, (double) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asDouble() < right.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + ClickHouseConstant converted = right.cast(ClickHouseDataType.Float64); + return this.asDouble() < converted.asDouble() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + + @Override + public boolean asBooleanNotNull() { + return Double.compare(value, 0.0) == 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Float64; + } + + @Override + public double asDouble() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant((long) value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant((long) value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant((long) value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant((long) value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant((long) value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant((long) value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf((long) value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf((long) value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java new file mode 100644 index 000000000..c388cb638 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt128Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt128Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt128Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int128; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java new file mode 100644 index 000000000..1fde4dfb1 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt16Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt16Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseInt16Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int16; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java new file mode 100644 index 000000000..f40ea5297 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt256Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt256Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt256Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int256; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java new file mode 100644 index 000000000..408df18fc --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt32Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt32Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseInt32Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int32; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java new file mode 100644 index 000000000..cb5c4b5f8 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt64Constant.java @@ -0,0 +1,131 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt64Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseInt64Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int64; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java new file mode 100644 index 000000000..38ad4878c --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseInt8Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseInt8Constant extends ClickHouseConstant { + + private final int value; + + public ClickHouseInt8Constant(int value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Int8; + } + + @Override + public boolean compareInternal(Object val) { + return value == (int) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java new file mode 100644 index 000000000..08370d560 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseNullConstant.java @@ -0,0 +1,53 @@ +package sqlancer.clickhouse.ast.constant; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseNullConstant extends ClickHouseConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public boolean asBooleanNotNull() { + throw new AssertionError(); + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.Nothing; + } + + @Override + public boolean compareInternal(Object value) { + return false; + } + + @Override + public ClickHouseConstant applyEquals(ClickHouseConstant right) { + return ClickHouseCreateConstant.createNullConstant(); + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + return ClickHouseCreateConstant.createNullConstant(); + } + + @Override + public Object getValue() { + return null; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + return null; + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java new file mode 100644 index 000000000..bde359f92 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java @@ -0,0 +1,125 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseStringConstant extends ClickHouseConstant { + + private final String value; + + public ClickHouseStringConstant(String value) { + this.value = value; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; + } + + @Override + public boolean asBooleanNotNull() { + return value.length() > 0; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((String) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asString().compareTo(right.asString()) <= 0 ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.String; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(Integer.parseInt(value)); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(Integer.parseInt(value)); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(Integer.parseInt(value)); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(Integer.parseInt(value)); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(Integer.parseInt(value)); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(Integer.parseInt(value)); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(Integer.parseInt(value))); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) Float.parseFloat(value)); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant((double) Double.parseDouble(value)); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value == "true"); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java new file mode 100644 index 000000000..5952c967c --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt128Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt128Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt128Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt128; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java new file mode 100644 index 000000000..c7ae84fdf --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt16Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt16Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseUInt16Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt16; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java new file mode 100644 index 000000000..95b333323 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt256Constant.java @@ -0,0 +1,139 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt256Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt256Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt256; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(value); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(value); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(value); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java new file mode 100644 index 000000000..8e4f5fd09 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt32Constant.java @@ -0,0 +1,130 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt32Constant extends ClickHouseConstant { + + private final long value; + + public ClickHouseUInt32Constant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt32; + } + + @Override + public boolean compareInternal(Object val) { + return value == (long) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java new file mode 100644 index 000000000..cd3363850 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt64Constant.java @@ -0,0 +1,131 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt64Constant extends ClickHouseConstant { + + private final BigInteger value; + + public ClickHouseUInt64Constant(BigInteger value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != BigInteger.ZERO; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt64; + } + + @Override + public boolean compareInternal(Object val) { + return value.compareTo((BigInteger) val) == 0; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value.longValueExact(); + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + long val = value.longValueExact(); + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(val); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(val); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(val); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(val); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(val); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(val); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(value); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(value); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant(value.floatValue()); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant(value.doubleValue()); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(val != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java new file mode 100644 index 000000000..de0d308c3 --- /dev/null +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseUInt8Constant.java @@ -0,0 +1,138 @@ +package sqlancer.clickhouse.ast.constant; + +import java.math.BigInteger; + +import com.clickhouse.client.ClickHouseDataType; + +import sqlancer.IgnoreMeException; +import sqlancer.clickhouse.ast.ClickHouseConstant; + +public class ClickHouseUInt8Constant extends ClickHouseConstant { + + private final int value; + + public ClickHouseUInt8Constant(int value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean isNull() { + return false; + } + + @Override + public boolean asBooleanNotNull() { + return value != 0; + } + + @Override + public ClickHouseDataType getDataType() { + return ClickHouseDataType.UInt8; + } + + @Override + public boolean compareInternal(Object val) { + return value == (int) val; + } + + @Override + public ClickHouseConstant applyLess(ClickHouseConstant right) { + if (this.getDataType() == right.getDataType()) { + return this.asInt() < right.asInt() ? ClickHouseCreateConstant.createTrue() + : ClickHouseCreateConstant.createFalse(); + } + throw new IgnoreMeException(); + } + + @Override + public long asInt() { + return value; + } + + @Override + public Object getValue() { + return value; + } + + @Override + public ClickHouseConstant cast(ClickHouseDataType type) { + switch (type) { + case String: + return ClickHouseCreateConstant.createStringConstant(this.toString()); + case UInt8: + return ClickHouseCreateConstant.createUInt8Constant(value); + case Int8: + return ClickHouseCreateConstant.createInt8Constant(value); + case UInt16: + return ClickHouseCreateConstant.createUInt16Constant(value); + case Int16: + return ClickHouseCreateConstant.createInt16Constant(value); + case UInt32: + return ClickHouseCreateConstant.createUInt32Constant(value); + case Int32: + return ClickHouseCreateConstant.createInt32Constant(value); + case UInt64: + return ClickHouseCreateConstant.createUInt64Constant(BigInteger.valueOf(value)); + case Int64: + return ClickHouseCreateConstant.createInt64Constant(BigInteger.valueOf(value)); + case UInt128: + return ClickHouseCreateConstant.createUInt128Constant(BigInteger.valueOf(value)); + case Int128: + return ClickHouseCreateConstant.createInt128Constant(BigInteger.valueOf(value)); + case UInt256: + return ClickHouseCreateConstant.createUInt256Constant(BigInteger.valueOf(value)); + case Int256: + return ClickHouseCreateConstant.createInt256Constant(BigInteger.valueOf(value)); + case Float32: + return ClickHouseCreateConstant.createFloat32Constant((float) value); + case Float64: + return ClickHouseCreateConstant.createFloat64Constant((double) value); + case Nothing: + return ClickHouseCreateConstant.createNullConstant(); + case Bool: + return ClickHouseCreateConstant.createBooleanConstant(value != 0); + case IntervalYear: + case IntervalQuarter: + case IntervalMonth: + case IntervalWeek: + case IntervalDay: + case IntervalHour: + case IntervalMinute: + case IntervalSecond: + case Date: + case Date32: + case DateTime: + case DateTime32: + case DateTime64: + case Decimal: + case Decimal32: + case Decimal64: + case Decimal128: + case Decimal256: + case UUID: + case Enum: + case Enum8: + case Enum16: + case IPv4: + case IPv6: + case FixedString: + case AggregateFunction: + case SimpleAggregateFunction: + case Array: + case Map: + case Nested: + case Tuple: + case Point: + case Polygon: + case MultiPolygon: + case Ring: + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java index 612e807d5..deae2a9aa 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseCommon.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseCommon.java @@ -29,7 +29,7 @@ public static List getTableRefs(List tableRefs = new ArrayList<>(); for (ClickHouseSchema.ClickHouseTable t : tables) { ClickHouseTableReference tableRef; - tableRef = new ClickHouseTableReference(t); + tableRef = new ClickHouseTableReference(t, null); tableRefs.add(tableRef); } return tableRefs; diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 6e86b28ec..cd8ad8ff3 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -4,22 +4,27 @@ import java.util.List; import java.util.stream.Collectors; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseBinaryArithmeticOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; +import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; -import sqlancer.clickhouse.ast.ClickHouseConstant; import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseTableReference; +import sqlancer.clickhouse.ast.ClickHouseUnaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.gen.TypedExpressionGenerator; public class ClickHouseExpressionGenerator @@ -28,12 +33,54 @@ public class ClickHouseExpressionGenerator private final ClickHouseGlobalState globalState; public boolean allowAggregateFunctions; + private final List columnRefs; + public ClickHouseExpressionGenerator(ClickHouseGlobalState globalState) { this.globalState = globalState; + this.columnRefs = new ArrayList<>(); + } + + public final void addColumns(List col) { + this.columnRefs.addAll(col); + } + + private enum ColumnLike { + UNARY_PREFIX, BINARY_ARITHMETIC, UNARY_FUNCTION, BINARY_FUNCTION } private enum Expression { - UNARY_POSTFIX, UNARY_PREFIX, BINARY_COMPARISON, BINARY_LOGICAL + UNARY_PREFIX, BINARY_ARITHMETIC, UNARY_FUNCTION, BINARY_FUNCTION, BINARY_LOGICAL, BINARY_COMPARISON, + UNARY_POSTFIX + } + + public ClickHouseExpression generateExpressionWithColumns(List columns, int depth) { + if (columns.isEmpty() || depth > 0 && Randomly.getBoolean()) { + return generateConstant(null); + } + + if (depth >= 2 || Randomly.getBoolean()) { + return columns.get((int) Randomly.getNotCachedInteger(0, columns.size() - 1)); + } + + ColumnLike expr = Randomly.fromOptions(ColumnLike.values()); + switch (expr) { + case UNARY_PREFIX: + return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, depth + 1), + ClickHouseUnaryPrefixOperator.MINUS); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, depth + 1), + generateExpressionWithColumns(columns, depth + 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, depth + 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, depth + 1), + generateExpressionWithColumns(columns, depth + 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); + default: + throw new AssertionError(expr); + } } @Override @@ -66,42 +113,52 @@ protected ClickHouseExpression generateExpression(ClickHouseLancerDataType type, return new ClickHouseBinaryLogicalOperation(generateExpression(leftLeafType, depth + 1), generateExpression(rightLeafType, depth + 1), ClickHouseBinaryLogicalOperation.ClickHouseBinaryLogicalOperator.getRandom()); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(leftLeafType, depth + 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation(generateExpression(leftLeafType, depth + 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation(generateExpression(leftLeafType, depth + 1), + generateExpression(leftLeafType, depth + 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); default: throw new AssertionError(expr); } } - protected ClickHouseExpression generateJoinClause(ClickHouseSchema.ClickHouseTable leftTable, - ClickHouseSchema.ClickHouseTable rightTable) { - ClickHouseExpression leftExpr = getColumnNameFromTable(leftTable); - ClickHouseExpression rightExpr = getColumnNameFromTable(rightTable); - return new ClickHouseBinaryComparisonOperation(leftExpr, rightExpr, - ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.EQUALS); + protected ClickHouseExpression.ClickHouseJoinOnClause generateJoinClause(ClickHouseTableReference leftTable, + ClickHouseTableReference rightTable) { + List leftColumns = leftTable.getColumnReferences(); + List rightColumns = rightTable.getColumnReferences(); + ClickHouseExpression leftExpr = generateExpressionWithColumns(leftColumns, 0); + ClickHouseExpression rightExpr = generateExpressionWithColumns(rightColumns, 0); + return new ClickHouseExpression.ClickHouseJoinOnClause(leftExpr, rightExpr); } @Override protected ClickHouseExpression generateColumn(ClickHouseLancerDataType type) { - if (columns.isEmpty()) { + if (columnRefs.isEmpty()) { return generateConstant(type); } - List filteredColumns = columns.stream() - .filter(c -> c.getType().getType().name().equals(type.getType().name())).collect(Collectors.toList()); - ClickHouseColumn column = filteredColumns.isEmpty() ? Randomly.fromList(columns) - : Randomly.fromList(filteredColumns); - return new ClickHouseColumnReference(column); + List filteredColumns = columnRefs.stream() + .filter(c -> c.getColumn().getType().getType().name().equals(type.getType().name())) + .collect(Collectors.toList()); + return filteredColumns.isEmpty() ? Randomly.fromList(columnRefs) : Randomly.fromList(filteredColumns); } protected ClickHouseExpression getColumnNameFromTable(ClickHouseSchema.ClickHouseTable table) { - if (columns.isEmpty()) { + if (columnRefs.isEmpty()) { return generateConstant(ClickHouseLancerDataType.getRandom()); } - List filteredColumns = columns.stream().filter(c -> c.getTable() == table) - .collect(Collectors.toList()); + List filteredColumns = columnRefs.stream() + .filter(c -> c.getColumn().getTable() == table).collect(Collectors.toList()); if (filteredColumns.isEmpty()) { return generateConstant(ClickHouseLancerDataType.getRandom()); } - ClickHouseColumn column = Randomly.fromList(filteredColumns); - return new ClickHouseColumnReference(column); + return Randomly.fromList(filteredColumns); } @Override @@ -109,29 +166,29 @@ protected ClickHouseLancerDataType getRandomType() { return ClickHouseLancerDataType.getRandom(); } - public List getRandomJoinClauses( + public List getRandomJoinClauses(ClickHouseTableReference left, List tables) { List joinStatements = new ArrayList<>(); if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } - if (Randomly.getBoolean() && tables.size() > 1) { + List leftTables = new ArrayList<>(); + leftTables.add(left); + if (Randomly.getBoolean() && !tables.isEmpty()) { int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); for (int i = 0; i < nrJoinClauses; i++) { - ClickHouseSchema.ClickHouseTable leftTable = Randomly.fromList(tables); - ClickHouseSchema.ClickHouseTable rightTable = Randomly.fromList(tables); - ClickHouseExpression joinClause = generateJoinClause(leftTable, rightTable); - ClickHouseExpression.ClickHouseJoin.JoinType options; - options = Randomly.fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); - if (options == ClickHouseExpression.ClickHouseJoin.JoinType.NATURAL) { - // NATURAL joins do not have an ON clause - joinClause = null; - } + ClickHouseTableReference leftTable = leftTables + .get((int) Randomly.getNotCachedInteger(0, leftTables.size() - 1)); + ClickHouseTableReference rightTable = new ClickHouseTableReference(Randomly.fromList(tables), + "right_" + i); + ClickHouseExpression.ClickHouseJoinOnClause joinClause = generateJoinClause(leftTable, rightTable); + ClickHouseExpression.ClickHouseJoin.JoinType options = Randomly + .fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(leftTable, rightTable, options, joinClause); joinStatements.add(j); + leftTables.add(rightTable); } - } return joinStatements; } @@ -142,7 +199,8 @@ protected boolean canGenerateColumnOfType(ClickHouseLancerDataType type) { } @Override - public ClickHouseExpression generateConstant(ClickHouseLancerDataType type) { + public ClickHouseExpression generateConstant(ClickHouseLancerDataType genType) { + ClickHouseLancerDataType type = (genType == null) ? ClickHouseLancerDataType.getRandom() : genType; switch (type.getType()) { case Int8: case UInt8: @@ -152,13 +210,13 @@ public ClickHouseExpression generateConstant(ClickHouseLancerDataType type) { case UInt32: case Int64: case UInt64: - return ClickHouseConstant.createIntConstant(type.getType(), globalState.getRandomly().getInteger()); + return ClickHouseCreateConstant.createIntConstant(type.getType(), globalState.getRandomly().getInteger()); case Float32: - return ClickHouseConstant.createFloat32Constant((float) globalState.getRandomly().getDouble()); + return ClickHouseCreateConstant.createFloat32Constant((float) globalState.getRandomly().getDouble()); case Float64: - return ClickHouseConstant.createFloat64Constant(globalState.getRandomly().getDouble()); + return ClickHouseCreateConstant.createFloat64Constant(globalState.getRandomly().getDouble()); case String: - return ClickHouseConstant.createStringConstant(globalState.getRandomly().getString()); + return ClickHouseCreateConstant.createStringConstant(globalState.getRandomly().getString()); default: throw new AssertionError(); } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index 5b62e88bc..f3e5baecc 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -2,8 +2,10 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; + +import com.clickhouse.client.ClickHouseDataType; -import ru.yandex.clickhouse.domain.ClickHouseDataType; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; @@ -79,8 +81,8 @@ public void start() { if (engine == ClickHouseEngine.MergeTree) { if (Randomly.getBoolean()) { sb.append(" ORDER BY "); - ClickHouseExpression expr = gen - .generateExpression(ClickHouseSchema.ClickHouseLancerDataType.getRandom()); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); sb.append(ClickHouseToStringVisitor.asString(expr)); } else { sb.append(" ORDER BY tuple() "); @@ -88,14 +90,14 @@ public void start() { if (Randomly.getBoolean()) { sb.append(" PARTITION BY "); - ClickHouseExpression expr = gen - .generateExpression(ClickHouseSchema.ClickHouseLancerDataType.getRandom()); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); sb.append(ClickHouseToStringVisitor.asString(expr)); } if (Randomly.getBoolean()) { sb.append(" SAMPLE BY "); - ClickHouseExpression expr = gen - .generateExpression(ClickHouseSchema.ClickHouseLancerDataType.getRandom()); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); sb.append(ClickHouseToStringVisitor.asString(expr)); } // TODO: PRIMARY KEY diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java index b8da925e5..c7d56ab99 100644 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -19,7 +19,6 @@ import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; import sqlancer.clickhouse.ast.ClickHouseTableReference; @@ -41,17 +40,17 @@ public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { @Override public void check() throws SQLException { + ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); + ClickHouseTable leftTable = randomTables.getTables().remove(0); + ClickHouseTableReference table = new ClickHouseTableReference(leftTable, "left"); List columns = randomTables.getColumns(); + columns.addAll(leftTable.getColumns()); ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); ClickHouseExpression randomWhereCondition = gen.generateExpression(ClickHouseLancerDataType.getRandom()); - List tables = randomTables.getTables(); - List tableList = tables.stream().map(t -> new ClickHouseTableReference(t)) - .collect(Collectors.toList()); - List joins = gen.getRandomJoinClauses(tables); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); - int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, - randomWhereCondition, joins); + List joins = gen.getRandomJoinClauses(table, randomTables.getTables()); + int secondCount = getSecondQuery(table, randomWhereCondition, joins); + int firstCount = getFirstQueryCount(con, table, columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { throw new IgnoreMeException(); } @@ -61,14 +60,14 @@ public void check() throws SQLException { } } - private int getSecondQuery(List tableList, ClickHouseExpression whereClause, + private int getSecondQuery(ClickHouseExpression table, ClickHouseExpression whereClause, List joins) throws SQLException { ClickHouseSelect select = new ClickHouseSelect(); ClickHouseExpression inner = new ClickHouseAliasOperation(whereClause, "check"); select.setFetchColumns(Arrays.asList(inner)); - select.setFromList(tableList); + select.setFromClause(table); select.setJoinClauses(joins); int secondCount = 0; unoptimizedQueryString = "SELECT SUM(check <> 0) FROM (" + ClickHouseToStringVisitor.asString(select) @@ -91,14 +90,14 @@ private int getSecondQuery(List tableList, ClickHouseExpre return secondCount; } - private int getFirstQueryCount(SQLConnection con, List tableList, - List columns, ClickHouseExpression randomWhereCondition, - List joins) throws SQLException { + private int getFirstQueryCount(SQLConnection con, ClickHouseExpression tableList, List columns, + ClickHouseExpression randomWhereCondition, List joins) + throws SQLException { ClickHouseSelect select = new ClickHouseSelect(); - List allColumns = columns.stream().map((c) -> new ClickHouseColumnReference(c)) + List allColumns = columns.stream().map(c -> c.asColumnReference(null)) .collect(Collectors.toList()); select.setFetchColumns(allColumns); - select.setFromList(tableList); + select.setFromClause(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { select.setOrderByExpressions( diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 6275b6a87..2646878a2 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -4,7 +4,8 @@ import java.util.Arrays; import java.util.List; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; + import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; @@ -12,17 +13,16 @@ import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; public class ClickHouseTLPAggregateOracle extends ClickHouseTLPBase { - private ClickHouseExpressionGenerator gen; - public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState state) { super(state); ClickHouseErrors.addExpectedExpressionErrors(errors); @@ -31,8 +31,15 @@ public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState sta @Override public void check() throws SQLException { ClickHouseSchema s = state.getSchema(); - ClickHouseSchema.ClickHouseTables targetTables = s.getRandomTableNonEmptyTables(); - gen = new ClickHouseExpressionGenerator(state).setColumns(targetTables.getColumns()); + ClickHouseSchema.ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); + ClickHouseSchema.ClickHouseTable table = randomTables.getTables().remove(0); + ClickHouseTableReference tableRef = new ClickHouseTableReference(table, table.getName()); + List columns = randomTables.getColumns(); + columns.addAll(table.getColumns()); + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); + List joins = gen.getRandomJoinClauses(tableRef, randomTables.getTables()); + + gen = new ClickHouseExpressionGenerator(state).setColumns(randomTables.getColumns()); ClickHouseSelect select = new ClickHouseSelect(); ClickHouseAggregate.ClickHouseAggregateFunction windowFunction = Randomly.fromOptions( ClickHouseAggregate.ClickHouseAggregateFunction.MIN, @@ -41,8 +48,9 @@ public void check() throws SQLException { ClickHouseAggregate aggregate = new ClickHouseAggregate( gen.generateExpressions(ClickHouseSchema.ClickHouseLancerDataType.getRandom(), 1), windowFunction); select.setFetchColumns(Arrays.asList(aggregate)); - List from = ClickHouseCommon.getTableRefs(targetTables.getTables(), s); - select.setFromList(from); + ClickHouseExpression from = tableRef; + select.setFromClause(from); + select.setJoinClauses(joins); if (Randomly.getBoolean()) { select.setOrderByExpressions(gen.generateOrderBys()); } @@ -56,12 +64,22 @@ public void check() throws SQLException { ClickHouseUnaryPostfixOperation notNullClause = new ClickHouseUnaryPostfixOperation(whereClause, ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.IS_NULL, false); - ClickHouseSelect leftSelect = getSelect(aggregate, from, whereClause); - ClickHouseSelect middleSelect = getSelect(aggregate, from, negatedClause); - ClickHouseSelect rightSelect = getSelect(aggregate, from, notNullClause); + select.setFetchColumns(Arrays.asList(new ClickHouseAliasOperation(aggregate, "aggr"))); + select.setFromClause(from); + select.setWhereClause(whereClause); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setGroupByClause(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + String metamorphicText = "SELECT " + aggregate.getFunc().toString() + "(aggr) FROM ("; - metamorphicText += ClickHouseVisitor.asString(leftSelect) + " UNION ALL " - + ClickHouseVisitor.asString(middleSelect) + " UNION ALL " + ClickHouseVisitor.asString(rightSelect); + metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; + select.setWhereClause(negatedClause); + metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; + select.setWhereClause(notNullClause); + metamorphicText += ClickHouseVisitor.asString(select); metamorphicText += ")"; metamorphicText += " SETTINGS aggregate_functions_null_for_empty = 1"; List firstResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQuery, errors, state); @@ -87,20 +105,4 @@ public void check() throws SQLException { } } - private ClickHouseSelect getSelect(ClickHouseAggregate aggregate, List from, - ClickHouseExpression whereClause) { - ClickHouseSelect leftSelect = new ClickHouseSelect(); - leftSelect.setFetchColumns( - Arrays.asList(new ClickHouseExpression.ClickHousePostfixText(aggregate, " as aggr", null))); - leftSelect.setFromList(from); - leftSelect.setWhereClause(whereClause); - if (Randomly.getBooleanWithRatherLowProbability()) { - leftSelect.setGroupByClause(gen.generateExpressions(Randomly.smallNumber() + 1)); - } - if (Randomly.getBoolean()) { - leftSelect.setOrderByExpressions(gen.generateOrderBys()); - } - return leftSelect; - } - } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 245343830..a9b09fd92 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -1,20 +1,25 @@ package sqlancer.clickhouse.oracle.tlp; +import static java.lang.Math.min; +import static java.util.stream.IntStream.range; + import java.sql.SQLException; import java.util.List; import java.util.stream.Collectors; +import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; +import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.gen.ClickHouseCommon; +import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; @@ -23,7 +28,7 @@ public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { - ClickHouseSchema s; + ClickHouseSchema schema; ClickHouseTables targetTables; ClickHouseExpressionGenerator gen; ClickHouseSelect select; @@ -35,23 +40,34 @@ public ClickHouseTLPBase(ClickHouseGlobalState state) { @Override public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new ClickHouseExpressionGenerator(state).setColumns(targetTables.getColumns()); + gen = new ClickHouseExpressionGenerator(state); + schema = state.getSchema(); initializeTernaryPredicateVariants(); select = new ClickHouseSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List joinStatements = gen.getRandomJoinClauses(tables); - List tableRefs = ClickHouseCommon.getTableRefs(tables, s); - select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); - select.setFromTables(tableRefs); + List tables = schema.getRandomTableNonEmptyTables().getTables(); + ClickHouseTableReference table = new ClickHouseTableReference( + tables.get((int) Randomly.getNotCachedInteger(0, tables.size())), + Randomly.getBoolean() ? "left" : null); + select.setFromClause(table); + List columns = table.getColumnReferences(); + + if (state.getClickHouseOptions().testJoins && Randomly.getBoolean()) { + List joinStatements = gen.getRandomJoinClauses(table, tables); + select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); + } + gen.addColumns(columns); + range(0, Randomly.smallNumber()).mapToObj(i -> gen.generateExpressionWithColumns(columns, 0)) + .collect(Collectors.toList()); + select.setFetchColumns(generateFetchColumns(columns)); select.setWhereClause(null); + // Smoke check + ComparatorHelper.getResultSetFirstColumnAsString(ClickHouseVisitor.asString(select), errors, state); } - List generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new ClickHouseColumnReference(c)) - .collect(Collectors.toList()); + List generateFetchColumns(List columns) { + List list = Randomly.extractNrRandomColumns(columns, + min(1 + Randomly.smallNumber(), columns.size())); + return list.stream().map(c -> (ClickHouseExpression) c).collect(Collectors.toList()); } @Override diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index 0cdee3864..8aff8fd77 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -3,14 +3,10 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import sqlancer.ComparatorHelper; -import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseVisitor; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; -import sqlancer.clickhouse.ast.ClickHouseExpression; public class ClickHouseTLPGroupByOracle extends ClickHouseTLPBase { @@ -39,13 +35,4 @@ public void check() throws SQLException { ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } - - @Override - List generateFetchColumns() { - List columns; - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new ClickHouseColumnReference(c)) - .collect(Collectors.toList()); - return columns; - } - } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index c7bde4b70..d71e17624 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -4,8 +4,10 @@ import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; @@ -14,9 +16,9 @@ import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.gen.ClickHouseCommon; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; public class ClickHouseTLPHavingOracle extends ClickHouseTLPBase { @@ -29,25 +31,36 @@ public ClickHouseTLPHavingOracle(ClickHouseProvider.ClickHouseGlobalState state) @Override public void check() throws SQLException { ClickHouseSchema s = state.getSchema(); - ClickHouseSchema.ClickHouseTables targetTables = s.getRandomTableNonEmptyTables(); - List groupByColumns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ClickHouseColumnReference(c)).collect(Collectors.toList()); - List columns = targetTables.getColumns(); + ClickHouseSchema.ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); + ClickHouseSchema.ClickHouseTable table = randomTables.getTables().remove(0); + ClickHouseTableReference tableRef = new ClickHouseTableReference(table, table.getName()); + List columns = randomTables.getColumns(); + columns.addAll(table.getColumns()); ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); + List joins = gen.getRandomJoinClauses(tableRef, randomTables.getTables()); + List colRefs = joins.stream() + .flatMap(j -> j.getRightTable().getColumnReferences().stream()).collect(Collectors.toList()); + ClickHouseExpressionGenerator aggrGen = new ClickHouseExpressionGenerator(state).allowAggregates(true) .setColumns(columns); ClickHouseSelect select = new ClickHouseSelect(); select.setFetchColumns(aggrGen.generateExpressions(Randomly.smallNumber() + 1)); - List tables = targetTables.getTables(); - List joinStatements = gen.getRandomJoinClauses(tables); - List from = ClickHouseCommon.getTableRefs(tables, state.getSchema()); - select.setJoinClauses(joinStatements); + ClickHouseTableReference from = new ClickHouseTableReference(table, table.getName()); + select.setJoinClauses(joins); select.setSelectType(ClickHouseSelect.SelectType.ALL); - select.setFromTables(from); + select.setFromClause(from); // TODO order by? + + List groupByColumns = IntStream.range(0, Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(colRefs, 0)).collect(Collectors.toList()); + + if (groupByColumns.isEmpty()) { + throw new IgnoreMeException(); + } select.setGroupByClause(groupByColumns); select.setHavingClause(null); String originalQueryString = ClickHouseVisitor.asString(select); + originalQueryString += " SETTINGS aggregate_functions_null_for_empty=1, enable_optimize_predicate_expression=0"; // https://github.com/ClickHouse/ClickHouse/issues/12264 List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); diff --git a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java index d9dd89a5f..97ca1e282 100644 --- a/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java +++ b/test/sqlancer/clickhouse/ast/ClickHouseBinaryComparisonOperationTest.java @@ -5,7 +5,8 @@ import java.util.Arrays; import java.util.stream.Collectors; -import ru.yandex.clickhouse.domain.ClickHouseDataType; +import com.clickhouse.client.ClickHouseDataType; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -13,117 +14,117 @@ class ClickHouseBinaryComparisonOperationTest { @Test void getExpectedValueTrueEqualsTrue() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant equals = trueConst.applyEquals(ClickHouseConstant.createTrue()); - assertEquals(equals.asInt(), 1); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant equals = trueConst.applyEquals(ClickHouseCreateConstant.createTrue()); + assertEquals(true, equals.asBooleanNotNull()); } @Test void getExpectedValueTrueNotEqualsFalse() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant falseConst = ClickHouseConstant.createFalse(); - ClickHouseConstant equals = trueConst.applyEquals(ClickHouseConstant.createFalse()); - ClickHouseConstant equalsFalse = falseConst.applyEquals(ClickHouseConstant.createTrue()); - assertEquals(equals.asInt(), 0); - assertEquals(equalsFalse.asInt(), 0); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant falseConst = ClickHouseCreateConstant.createFalse(); + ClickHouseConstant equals = trueConst.applyEquals(ClickHouseCreateConstant.createFalse()); + ClickHouseConstant equalsFalse = falseConst.applyEquals(ClickHouseCreateConstant.createTrue()); + assertEquals(false, equals.asBooleanNotNull()); + assertEquals(false, equalsFalse.asBooleanNotNull()); } @Test void getExpectedValueFloat64EqualsFloat64() { - ClickHouseConstant oneConst = ClickHouseConstant.createFloat64Constant(1); - ClickHouseConstant oneFConst = ClickHouseConstant.createFloat64Constant(1.0); - ClickHouseConstant zeroConst = ClickHouseConstant.createFloat64Constant(0); - ClickHouseConstant zeroFConst = ClickHouseConstant.createFloat64Constant(0.0); - - assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(oneFConst.applyEquals(oneFConst).asInt(), 1); - assertEquals(oneConst.applyEquals(oneFConst).asInt(), 1); - - assertEquals(oneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(oneFConst.applyEquals(zeroFConst).asInt(), 0); - assertEquals(zeroConst.applyEquals(zeroFConst).asInt(), 1); - assertEquals(zeroFConst.applyEquals(zeroConst).asInt(), 1); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createFloat64Constant(1); + ClickHouseConstant oneFConst = ClickHouseCreateConstant.createFloat64Constant(1.0); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createFloat64Constant(0); + ClickHouseConstant zeroFConst = ClickHouseCreateConstant.createFloat64Constant(0.0); + + assertEquals(true, oneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(true, oneFConst.applyEquals(oneFConst).asBooleanNotNull()); + assertEquals(true, oneConst.applyEquals(oneFConst).asBooleanNotNull()); + + assertEquals(false, oneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, oneFConst.applyEquals(zeroFConst).asBooleanNotNull()); + assertEquals(true, zeroConst.applyEquals(zeroFConst).asBooleanNotNull()); + assertEquals(true, zeroFConst.applyEquals(zeroConst).asBooleanNotNull()); } @Test void getExpectedValueInt32EqualsBool() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant falseConst = ClickHouseConstant.createFalse(); - ClickHouseConstant oneConst = ClickHouseConstant.createInt32Constant(1); - ClickHouseConstant zeroConst = ClickHouseConstant.createInt32Constant(0); - ClickHouseConstant negativeConst = ClickHouseConstant.createInt32Constant(-100); - ClickHouseConstant positiveConst = ClickHouseConstant.createInt32Constant(10000); - - assertEquals(trueConst.applyEquals(oneConst).asInt(), 1); - assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(falseConst.applyEquals(oneConst).asInt(), 0); - - assertEquals(trueConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(oneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(falseConst.applyEquals(zeroConst).asInt(), 1); - - assertEquals(negativeConst.applyEquals(oneConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(trueConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(falseConst).asInt(), 0); - - assertEquals(positiveConst.applyEquals(oneConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(trueConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(falseConst).asInt(), 0); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant falseConst = ClickHouseCreateConstant.createFalse(); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createInt32Constant(1); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createInt32Constant(0); + ClickHouseConstant negativeConst = ClickHouseCreateConstant.createInt32Constant(-100); + ClickHouseConstant positiveConst = ClickHouseCreateConstant.createInt32Constant(10000); + + assertEquals(true, trueConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(true, oneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, falseConst.applyEquals(oneConst).asBooleanNotNull()); + + assertEquals(false, trueConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, oneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(true, falseConst.applyEquals(zeroConst).asBooleanNotNull()); + + assertEquals(false, negativeConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(falseConst).asBooleanNotNull()); + + assertEquals(false, positiveConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(falseConst).asBooleanNotNull()); } @Test void getExpectedValueIntEqualsInt() { - ClickHouseConstant trueConst = ClickHouseConstant.createTrue(); - ClickHouseConstant falseConst = ClickHouseConstant.createFalse(); + ClickHouseConstant trueConst = ClickHouseCreateConstant.createTrue(); + ClickHouseConstant falseConst = ClickHouseCreateConstant.createFalse(); for (ClickHouseDataType type : Arrays. stream(ClickHouseDataType.values()) .filter((dt) -> dt.name().contains("Int") && !dt.name().contains("Interval")) .collect(Collectors.toList())) { - ClickHouseConstant oneConst = ClickHouseConstant.createIntConstant(type, 1); - ClickHouseConstant zeroConst = ClickHouseConstant.createIntConstant(type, 0); - ClickHouseConstant negativeConst = ClickHouseConstant.createIntConstant(type, -100); - ClickHouseConstant positiveConst = ClickHouseConstant.createIntConstant(type, 10000); - - assertEquals(trueConst.applyEquals(oneConst).asInt(), 1); - assertEquals(oneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(falseConst.applyEquals(oneConst).asInt(), 0); - - assertEquals(trueConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(oneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(falseConst.applyEquals(zeroConst).asInt(), 1); - - assertEquals(negativeConst.applyEquals(oneConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(trueConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(falseConst).asInt(), 0); - - assertEquals(positiveConst.applyEquals(oneConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(trueConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(falseConst).asInt(), 0); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createIntConstant(type, 1); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createIntConstant(type, 0); + ClickHouseConstant negativeConst = ClickHouseCreateConstant.createIntConstant(type, -100); + ClickHouseConstant positiveConst = ClickHouseCreateConstant.createIntConstant(type, 10000); + + assertEquals(true, trueConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(true, oneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, falseConst.applyEquals(oneConst).asBooleanNotNull()); + + assertEquals(false, trueConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, oneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(true, falseConst.applyEquals(zeroConst).asBooleanNotNull()); + + assertEquals(false, negativeConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(falseConst).asBooleanNotNull()); + + assertEquals(false, positiveConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(trueConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(falseConst).asBooleanNotNull()); } } @Test void getExpectedValueInt32EqualsFloat64() { - ClickHouseConstant float64OneConst = ClickHouseConstant.createFloat64Constant(1.0); - ClickHouseConstant float64ZeroConst = ClickHouseConstant.createFloat64Constant(0.0); - ClickHouseConstant oneConst = ClickHouseConstant.createInt32Constant(1); - ClickHouseConstant zeroConst = ClickHouseConstant.createInt32Constant(0); - ClickHouseConstant negativeConst = ClickHouseConstant.createInt32Constant(-100); - ClickHouseConstant positiveConst = ClickHouseConstant.createInt32Constant(10000); + ClickHouseConstant float64OneConst = ClickHouseCreateConstant.createFloat64Constant(1.0); + ClickHouseConstant float64ZeroConst = ClickHouseCreateConstant.createFloat64Constant(0.0); + ClickHouseConstant oneConst = ClickHouseCreateConstant.createInt32Constant(1); + ClickHouseConstant zeroConst = ClickHouseCreateConstant.createInt32Constant(0); + ClickHouseConstant negativeConst = ClickHouseCreateConstant.createInt32Constant(-100); + ClickHouseConstant positiveConst = ClickHouseCreateConstant.createInt32Constant(10000); - assertEquals(float64OneConst.applyEquals(oneConst).asInt(), 1); - assertEquals(float64ZeroConst.applyEquals(oneConst).asInt(), 0); + assertEquals(true, float64OneConst.applyEquals(oneConst).asBooleanNotNull()); + assertEquals(false, float64ZeroConst.applyEquals(oneConst).asBooleanNotNull()); - assertEquals(float64OneConst.applyEquals(zeroConst).asInt(), 0); - assertEquals(float64ZeroConst.applyEquals(zeroConst).asInt(), 1); + assertEquals(false, float64OneConst.applyEquals(zeroConst).asBooleanNotNull()); + assertEquals(true, float64ZeroConst.applyEquals(zeroConst).asBooleanNotNull()); - assertEquals(negativeConst.applyEquals(float64OneConst).asInt(), 0); - assertEquals(negativeConst.applyEquals(float64ZeroConst).asInt(), 0); + assertEquals(false, negativeConst.applyEquals(float64OneConst).asBooleanNotNull()); + assertEquals(false, negativeConst.applyEquals(float64ZeroConst).asBooleanNotNull()); - assertEquals(positiveConst.applyEquals(float64OneConst).asInt(), 0); - assertEquals(positiveConst.applyEquals(float64ZeroConst).asInt(), 0); + assertEquals(false, positiveConst.applyEquals(float64OneConst).asBooleanNotNull()); + assertEquals(false, positiveConst.applyEquals(float64ZeroConst).asBooleanNotNull()); } } diff --git a/test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java b/test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java new file mode 100644 index 000000000..fc786a625 --- /dev/null +++ b/test/sqlancer/clickhouse/ast/ClickHouseOperatorsVisitorTest.java @@ -0,0 +1,117 @@ +package sqlancer.clickhouse.ast; + +import org.junit.jupiter.api.Test; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; + +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ClickHouseOperatorsVisitorTest { + + @Test + void selectUnaryNot() { + ClickHouseConstant trueConst = ClickHouseCreateConstant.createBoolean(true); + ClickHouseExpression notTrue = new ClickHouseUnaryPrefixOperation(trueConst, + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.NOT); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(notTrue)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT (NOT (true))"; + assertEquals(answer, result); + } + + @Test + void selectUnaryMinus() { + ClickHouseConstant fiveConst = ClickHouseCreateConstant.createUInt32Constant(5); + ClickHouseExpression minusFive = new ClickHouseUnaryPrefixOperation(fiveConst, + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.MINUS); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(minusFive)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT (- (5))"; + assertEquals(answer, result); + } + + @Test + void selectUnaryExp() { + ClickHouseConstant tenConst = ClickHouseCreateConstant.createInt32Constant(10); + ClickHouseExpression minusFive = new ClickHouseUnaryFunctionOperation(tenConst, + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.EXP); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(minusFive)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT (exp (10))"; + assertEquals(answer, result); + } + + @Test + void selectBinaryPlus() { + ClickHouseConstant dConst = ClickHouseCreateConstant.createFloat32Constant((float) -1.1); + ClickHouseConstant tenConst = ClickHouseCreateConstant.createInt32Constant(10); + ClickHouseExpression expr = new ClickHouseBinaryArithmeticOperation(dConst, tenConst, + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.ADD); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT ((-1.1)+(10))"; + assertEquals(answer, result); + } + + @Test + void selectBinaryPow() { + ClickHouseConstant threeConst = ClickHouseCreateConstant.createInt8Constant(3); + ClickHouseConstant tenConst = ClickHouseCreateConstant.createInt32Constant(10); + ClickHouseExpression expr = new ClickHouseBinaryFunctionOperation(threeConst, tenConst, + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.POW); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT pow(3,10)"; + assertEquals(answer, result); + } + + @Test + void selectBinaryLCM() { + ClickHouseConstant aConst = ClickHouseCreateConstant.createInt8Constant(100); + ClickHouseConstant bConst = ClickHouseCreateConstant.createInt32Constant(-100); + ClickHouseExpression expr = new ClickHouseBinaryFunctionOperation(aConst, bConst, + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.LCM); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT lcm(100,-100)"; + assertEquals(answer, result); + } + + @Test + void selectBinaryDivCol() { + ClickHouseColumnReference a = new ClickHouseColumnReference(new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null), null, null); + ClickHouseColumnReference b = new ClickHouseColumnReference(new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null), null, null); + ClickHouseExpression expr = new ClickHouseBinaryFunctionOperation(a, b, + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.INT_DIV); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT intDiv(a,b)"; + assertEquals(answer, result); + } + + @Test + void selectBinaryComp() { + ClickHouseConstant aConst = ClickHouseCreateConstant.createInt8Constant(10); + ClickHouseConstant bConst = ClickHouseCreateConstant.createInt32Constant(100); + ClickHouseExpression expr = new ClickHouseBinaryComparisonOperation(aConst, bConst, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.GREATER); + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(expr)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT ((10)>(100))"; + assertEquals(answer, result); + } + +} diff --git a/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java new file mode 100644 index 000000000..7231aced8 --- /dev/null +++ b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java @@ -0,0 +1,345 @@ +package sqlancer.clickhouse.ast; + +import org.junit.jupiter.api.Test; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.constant.ClickHouseInt8Constant; +import sqlancer.common.schema.TableIndex; + +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class ClickHouseToStringVisitorTest { + + @Test + void select1Test() { + ClickHouseConstant oneConst = new ClickHouseInt8Constant(1); + ClickHouseSelect selectOne = new ClickHouseSelect(); + selectOne.setFetchColumns(Arrays.asList(oneConst)); + String result = ClickHouseVisitor.asString(selectOne); + String answer = "SELECT 1"; + assertEquals(answer, result); + } + + @Test + void select1asATest() { + ClickHouseAliasOperation oneConstAsA = new ClickHouseAliasOperation(new ClickHouseInt8Constant(1), "a"); + ClickHouseSelect selectOne = new ClickHouseSelect(); + selectOne.setFetchColumns(Arrays.asList(oneConstAsA)); + String result = ClickHouseVisitor.asString(selectOne); + String answer = "SELECT 1 AS `a`"; + assertEquals(answer, result); + } + + @Test + void selectATest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseSelect selectA = new ClickHouseSelect(); + selectA.setFetchColumns(Arrays.asList(a_ref)); + selectA.setFromClause(table_ref); + String result = ClickHouseVisitor.asString(selectA); + String answer = "SELECT t.a FROM t"; + assertEquals(answer, result); + } + + @Test + void selectAasBTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseAliasOperation b = new ClickHouseAliasOperation(a_ref, "b"); + ClickHouseColumnReference b_ref = new ClickHouseColumnReference(b); + ClickHouseSelect selectA = new ClickHouseSelect(); + selectA.setFetchColumns(Arrays.asList(b, b_ref)); + selectA.setFromClause(table_ref); + String result = ClickHouseVisitor.asString(selectA); + String answer = "SELECT t.a AS `b`, b FROM t"; + assertEquals(answer, result); + } + + @Test + void selectABTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + selectAB.setFetchColumns(Arrays.asList(a_ref, b_ref)); + selectAB.setFromClause(table_ref); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT t.a, t.b FROM t"; + assertEquals(answer, result); + } + + @Test + void selectWhereAGreaterBTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + selectAB.setFetchColumns(Arrays.asList(a_ref, b_ref)); + selectAB.setFromClause(table_ref); + selectAB.setWhereClause(new ClickHouseBinaryComparisonOperation(a_ref, b_ref, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.GREATER)); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT t.a, t.b FROM t WHERE ((t.a)>(t.b))"; + assertEquals(answer, result); + } + + @Test + void selectWhereAGreaterConstTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + selectAB.setFetchColumns(Arrays.asList(a_ref, b_ref)); + selectAB.setFromClause(table_ref); + ClickHouseConstant c_const = new ClickHouseInt8Constant(1); + selectAB.setWhereClause(new ClickHouseBinaryComparisonOperation(a_ref, c_const, + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.GREATER)); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT t.a, t.b FROM t WHERE ((t.a)>(1))"; + assertEquals(answer, result); + } + + @Test + void selectSumAGroupByBTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table = new ClickHouseSchema.ClickHouseTable("t", empty_col_list, indexes, + false); + ClickHouseTableReference table_ref = new ClickHouseTableReference(table, null); + ClickHouseSchema.ClickHouseColumn a_col = new ClickHouseSchema.ClickHouseColumn("a", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + ClickHouseSchema.ClickHouseColumn b_col = new ClickHouseSchema.ClickHouseColumn("b", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table); + a_col.setTable(table); + b_col.setTable(table); + ClickHouseColumnReference a_ref = a_col.asColumnReference(null); + ClickHouseColumnReference b_ref = b_col.asColumnReference(null); + ClickHouseSelect selectAB = new ClickHouseSelect(); + ClickHouseAggregate sum_a = new ClickHouseAggregate(Arrays.asList(a_ref), + ClickHouseAggregate.ClickHouseAggregateFunction.SUM); + selectAB.setFetchColumns(Arrays.asList(sum_a)); + selectAB.setFromClause(table_ref); + selectAB.setGroupByClause(Arrays.asList(b_ref)); + String result = ClickHouseVisitor.asString(selectAB); + String answer = "SELECT SUM(t.a) FROM t GROUP BY t.b"; + assertEquals(answer, result); + } + + @Test + void selectCrossJoinTest() { + List empty_col_list = Collections.emptyList(); + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", empty_col_list, indexes, + false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", empty_col_list, indexes, + false); + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, null); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, null); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table1); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table1); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table2); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, table2); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseColumnReference a1_ref = a1_col.asColumnReference(null); + ClickHouseColumnReference b1_ref = b1_col.asColumnReference(null); + ClickHouseColumnReference a2_ref = a2_col.asColumnReference(null); + ClickHouseColumnReference b2_ref = b2_col.asColumnReference(null); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.CROSS); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT t1.a1, t2.a2, t1.b1, t2.b2 FROM t1 JOIN t2"; + assertEquals(answer, result); + } + + @Test + void selectCrossJoinAliasedTest() { + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", + Arrays.asList(a1_col, b1_col), indexes, false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", + Arrays.asList(a2_col, b2_col), indexes, false); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, "left"); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, "right"); + + List t1_col_ref = table1_ref.getColumnReferences(); + ClickHouseColumnReference a1_ref = t1_col_ref.get(0); + ClickHouseColumnReference a2_ref = t1_col_ref.get(1); + + List t2_col_ref = table2_ref.getColumnReferences(); + ClickHouseColumnReference b1_ref = t2_col_ref.get(0); + ClickHouseColumnReference b2_ref = t2_col_ref.get(1); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.CROSS); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT left.a1, left.b1, right.a2, right.b2 FROM t1 AS left JOIN t2 AS right"; + assertEquals(answer, result); + } + + @Test + void selectJoinONTest() { + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", + Arrays.asList(a1_col, b1_col), indexes, false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", + Arrays.asList(a2_col, b2_col), indexes, false); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, null); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, null); + + List t1_col_ref = table1_ref.getColumnReferences(); + ClickHouseColumnReference a1_ref = t1_col_ref.get(0); + ClickHouseColumnReference b1_ref = t1_col_ref.get(1); + + List t2_col_ref = table2_ref.getColumnReferences(); + ClickHouseColumnReference a2_ref = t2_col_ref.get(0); + ClickHouseColumnReference b2_ref = t2_col_ref.get(1); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoinOnClause on = new ClickHouseExpression.ClickHouseJoinOnClause(a1_ref, + a2_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.INNER, on); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT t1.a1, t2.a2, t1.b1, t2.b2 FROM t1 INNER JOIN t2 ON ((t1.a1)=(t2.a2))"; + assertEquals(answer, result); + } + + @Test + void selectJoinONAliasedTest() { + List indexes = Collections.emptyList(); + ClickHouseSchema.ClickHouseColumn a1_col = new ClickHouseSchema.ClickHouseColumn("a1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b1_col = new ClickHouseSchema.ClickHouseColumn("b1", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn a2_col = new ClickHouseSchema.ClickHouseColumn("a2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseColumn b2_col = new ClickHouseSchema.ClickHouseColumn("b2", + ClickHouseSchema.ClickHouseLancerDataType.getRandom(), false, false, null); + ClickHouseSchema.ClickHouseTable table1 = new ClickHouseSchema.ClickHouseTable("t1", + Arrays.asList(a1_col, b1_col), indexes, false); + ClickHouseSchema.ClickHouseTable table2 = new ClickHouseSchema.ClickHouseTable("t2", + Arrays.asList(a2_col, b2_col), indexes, false); + a1_col.setTable(table1); + b1_col.setTable(table1); + a2_col.setTable(table2); + b2_col.setTable(table2); + + ClickHouseTableReference table1_ref = new ClickHouseTableReference(table1, "left"); + ClickHouseTableReference table2_ref = new ClickHouseTableReference(table2, "right"); + + List t1_col_ref = table1_ref.getColumnReferences(); + ClickHouseColumnReference a1_ref = t1_col_ref.get(0); + ClickHouseColumnReference b1_ref = t1_col_ref.get(1); + + List t2_col_ref = table2_ref.getColumnReferences(); + ClickHouseColumnReference a2_ref = t2_col_ref.get(0); + ClickHouseColumnReference b2_ref = t2_col_ref.get(1); + + ClickHouseSelect select = new ClickHouseSelect(); + select.setFetchColumns(Arrays.asList(a1_ref, a2_ref, b1_ref, b2_ref)); + select.setFromClause(table1_ref); + ClickHouseExpression.ClickHouseJoinOnClause on = new ClickHouseExpression.ClickHouseJoinOnClause(a1_ref, + a2_ref); + ClickHouseExpression.ClickHouseJoin join = new ClickHouseExpression.ClickHouseJoin(table1_ref, table2_ref, + ClickHouseExpression.ClickHouseJoin.JoinType.INNER, on); + select.setJoinClauses(Arrays.asList(join)); + String result = ClickHouseVisitor.asString(select); + String answer = "SELECT left.a1, right.a2, left.b1, right.b2 FROM t1 AS left INNER JOIN t2 AS right ON ((left.a1)=(right.a2))"; + assertEquals(answer, result); + } +} From 894b358b08a8059ad05eb51d0769205503ebf5bb Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 19 Nov 2022 12:28:00 +0800 Subject: [PATCH 237/856] tidb: ignore Empty pattern is invalid error (#616) Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index f37bdb685..0586846fc 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -31,6 +31,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { // regex errors.add("error parsing regexp"); errors.add("from regexp"); + errors.add("Empty pattern is invalid"); // To avoid bugs errors.add("Unknown column"); // https://github.com/pingcap/tidb/issues/35522 From 032a4460bf3c1d1da5861ae5dfb4195d27bc4e70 Mon Sep 17 00:00:00 2001 From: taichong Date: Sun, 20 Nov 2022 10:39:23 +0800 Subject: [PATCH 238/856] fix: add databend expr errors --- src/sqlancer/databend/DatabendErrors.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 7e1f80c4e..bea8c9558 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -9,6 +9,7 @@ private DatabendErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE @@ -21,11 +22,13 @@ public static void addExpressionErrors(ExpectedErrors errors) { public static void addInsertErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); } public static void addGroupByErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); } From 422b41ad185305d3b0ec318c059ad5f4d4fbdf36 Mon Sep 17 00:00:00 2001 From: Nishita Date: Wed, 23 Nov 2022 19:28:41 +0530 Subject: [PATCH 239/856] First commit for HSQLDB --- pom.xml | 6 + src/sqlancer/hsqldb/HSQLDBErrors.java | 24 ++ src/sqlancer/hsqldb/HSQLDBOptions.java | 31 +++ src/sqlancer/hsqldb/HSQLDBProvider.java | 104 ++++++++ src/sqlancer/hsqldb/HSQLDBSchema.java | 153 ++++++++++++ .../hsqldb/HSQLDBToStringVisitor.java | 91 +++++++ src/sqlancer/hsqldb/ast/HSQLDBConstant.java | 229 ++++++++++++++++++ src/sqlancer/hsqldb/ast/HSQLDBExpression.java | 4 + src/sqlancer/hsqldb/ast/HSQLDBJoin.java | 124 ++++++++++ src/sqlancer/hsqldb/ast/HSQLDBSelect.java | 18 ++ .../hsqldb/gen/HSQLDBExpressionGenerator.java | 88 +++++++ .../hsqldb/gen/HSQLDBInsertGenerator.java | 49 ++++ .../hsqldb/gen/HSQLDBTableGenerator.java | 57 +++++ .../hsqldb/gen/HSQLDBUpdateGenerator.java | 42 ++++ .../test/HSQLDBQueryPartitioningBase.java | 25 ++ 15 files changed, 1045 insertions(+) create mode 100644 src/sqlancer/hsqldb/HSQLDBErrors.java create mode 100644 src/sqlancer/hsqldb/HSQLDBOptions.java create mode 100644 src/sqlancer/hsqldb/HSQLDBProvider.java create mode 100644 src/sqlancer/hsqldb/HSQLDBSchema.java create mode 100644 src/sqlancer/hsqldb/HSQLDBToStringVisitor.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBConstant.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBExpression.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBJoin.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBSelect.java create mode 100644 src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java create mode 100644 src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java create mode 100644 src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java create mode 100644 src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java create mode 100644 src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java diff --git a/pom.xml b/pom.xml index 6dae29f62..ffee5dfd9 100644 --- a/pom.xml +++ b/pom.xml @@ -352,6 +352,12 @@ questdb 6.5.3 + + org.hsqldb + hsqldb + 2.7.1 + runtime + diff --git a/src/sqlancer/hsqldb/HSQLDBErrors.java b/src/sqlancer/hsqldb/HSQLDBErrors.java new file mode 100644 index 000000000..5fa77051a --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBErrors.java @@ -0,0 +1,24 @@ +package sqlancer.hsqldb; + +import sqlancer.common.query.ExpectedErrors; + +public final class HSQLDBErrors { + + private HSQLDBErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + } + + private static void addRegexErrors(ExpectedErrors errors) { + } + + private static void addFunctionErrors(ExpectedErrors errors) { + } + + public static void addInsertErrors(ExpectedErrors errors) { + addRegexErrors(errors); + addFunctionErrors(errors); + } + +} diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java new file mode 100644 index 000000000..b2418fe8d --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -0,0 +1,31 @@ +package sqlancer.hsqldb; + +import com.beust.jcommander.Parameter; +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.hsqldb.test.HSQLDBQueryPartitioningBase; + +import java.sql.SQLException; +import java.util.List; + +public class HSQLDBOptions implements DBMSSpecificOptions { + + @Parameter(names = "--oracle") + public List oracle = List.of(HSQLDBOracleFactory.WHERE); + + public enum HSQLDBOracleFactory implements OracleFactory { + WHERE { + @Override + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) throws SQLException { + return new HSQLDBQueryPartitioningBase(globalState); + } + } + } + + @Override + public List getTestOracleFactory() { + return oracle; + } + +} diff --git a/src/sqlancer/hsqldb/HSQLDBProvider.java b/src/sqlancer/hsqldb/HSQLDBProvider.java new file mode 100644 index 000000000..cd238df23 --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBProvider.java @@ -0,0 +1,104 @@ +package sqlancer.hsqldb; + +import com.google.auto.service.AutoService; +import sqlancer.*; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.hsqldb.gen.HSQLDBInsertGenerator; +import sqlancer.hsqldb.gen.HSQLDBTableGenerator; +import sqlancer.hsqldb.gen.HSQLDBUpdateGenerator; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +@AutoService(DatabaseProvider.class) +public class HSQLDBProvider extends SQLProviderAdapter { + + private static final String HSQLDB = "hsqldb"; + + public HSQLDBProvider() { + super(HSQLDBGlobalState.class, HSQLDBOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(HSQLDBInsertGenerator::getQuery), + UPDATE(HSQLDBUpdateGenerator::getQuery); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + @Override + public SQLConnection createDatabase(HSQLDBGlobalState globalState) throws Exception { + String databaseName = globalState.getDatabaseName(); + String url = "jdbc:hsqldb:file:" + databaseName; + MainOptions options = globalState.getOptions(); + Connection connection = DriverManager.getConnection(url, options.getUserName(), options.getPassword()); + //When a server instance is started, or when a connection is made to an in-process database, + // a new, empty database is created if no database exists at the given path. + try (Statement s = connection.createStatement()) { + s.execute("DROP SCHEMA PUBLIC CASCADE"); + s.execute("SET DATABASE SQL DOUBLE NAN FALSE"); + } + return new SQLConnection(connection); + } + + @Override + public String getDBMSName() { + return HSQLDB; + } + + @Override + public void generateDatabase(HSQLDBGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new HSQLDBTableGenerator().getQuery(globalState, null); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + StatementExecutor se = new StatementExecutor<>(globalState, HSQLDBProvider.Action.values(), + HSQLDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + private static int mapActions(HSQLDBProvider.HSQLDBGlobalState globalState, HSQLDBProvider.Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case UPDATE: + return r.getInteger(0, 1); + default: + throw new AssertionError(a); + } + } + + public static class HSQLDBGlobalState extends SQLGlobalState { + + @Override + protected HSQLDBSchema readSchema() throws SQLException { + return HSQLDBSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } +} + + diff --git a/src/sqlancer/hsqldb/HSQLDBSchema.java b/src/sqlancer/hsqldb/HSQLDBSchema.java new file mode 100644 index 000000000..13059ef3e --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBSchema.java @@ -0,0 +1,153 @@ +package sqlancer.hsqldb; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class HSQLDBSchema extends AbstractSchema { + + public HSQLDBSchema(List databaseTables) { + super(databaseTables); + } + + public static HSQLDBSchema fromConnection(SQLConnection connection, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(connection); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(connection, tableName); + boolean isView = tableName.startsWith("v"); + HSQLDBSchema.HSQLDBTable t = new HSQLDBSchema.HSQLDBTable(tableName, databaseColumns, isView); + for (HSQLDBSchema.HSQLDBColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new HSQLDBSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'PUBLIC';")) { + while (rs.next()) { + tableNames.add(rs.getString("TABLE_NAME")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + String sql = "SELECT COLUMN_NAME, DATA_TYPE, TYPE_NAME, COLUMN_SIZE FROM INFORMATION_SCHEMA.SYSTEM_COLUMNS WHERE TABLE_NAME = '%s';"; + try (ResultSet rs = s.executeQuery(String.format(sql, tableName))) { + while (rs.next()) { + HSQLDBDataType dataType = HSQLDBDataType.from(rs.getString("TYPE_NAME")); + HSQLDBCompositeDataType compositeDataType = new HSQLDBCompositeDataType(dataType, rs.getInt("COLUMN_SIZE")); + HSQLDBColumn column = new HSQLDBColumn(rs.getString("COLUMN_NAME"), null, compositeDataType ); + tableNames.add(column); + } + } + } + return tableNames; + } + + public static class HSQLDBTable extends AbstractRelationalTable { + + public HSQLDBTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + + public static class HSQLDBColumn extends AbstractTableColumn + { + + public HSQLDBColumn(String name, HSQLDBTable table, HSQLDBCompositeDataType type) { + super(name, table, type); + } + } + + public enum HSQLDBDataType { + + INTEGER, DOUBLE, BOOLEAN, CHAR, VARCHAR, BINARY, TIME, DATE, TIMESTAMP, NULL; + + public static HSQLDBSchema.HSQLDBDataType getRandomWithoutNull() { + HSQLDBSchema.HSQLDBDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == HSQLDBSchema.HSQLDBDataType.NULL); + return dt; + } + + public static HSQLDBDataType from(String type_name) { + for( HSQLDBDataType value : HSQLDBDataType.values()) { + if(value.name().equals(type_name)) { + return value; + } + } + return NULL; + } + } + + public static class HSQLDBCompositeDataType{ + private final int size; + private final HSQLDBDataType type; + + public HSQLDBCompositeDataType(HSQLDBDataType type, int size) { + this.type = type; + this.size = size; + } + + + public static HSQLDBCompositeDataType getRandomWithoutNull() { + HSQLDBSchema.HSQLDBDataType type = HSQLDBSchema.HSQLDBDataType.getRandomWithoutNull(); + int size; + switch (type) { + case VARCHAR: + case CHAR: + case TIME: + case BINARY: + case TIMESTAMP: + size = Randomly.fromOptions(4,6,8); + break; + case BOOLEAN: + case INTEGER: + case DOUBLE: + //case UUID: + //case OTHER: + case DATE: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new HSQLDBSchema.HSQLDBCompositeDataType(type, size); + } + + public HSQLDBDataType getType() { + return type; + } + + public int getSize() { + return size; + } + } +} diff --git a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java new file mode 100644 index 000000000..66ccbfd12 --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java @@ -0,0 +1,91 @@ +package sqlancer.hsqldb; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.hsqldb.ast.HSQLDBConstant; +import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; + +public class HSQLDBToStringVisitor extends NewToStringVisitor { + + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof HSQLDBConstant) { + visit((HSQLDBConstant) expr); + } else if (expr instanceof HSQLDBSelect) { + visit((HSQLDBSelect) expr); + } else if (expr instanceof HSQLDBJoin) { + visit((HSQLDBJoin) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + public static String asString(Node expr) { + HSQLDBToStringVisitor visitor = new HSQLDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + private void visit(HSQLDBJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getOuterType() != null) { + sb.append(join.getOuterType()); + } + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(HSQLDBConstant constant) { + sb.append(constant.toString()); + } + + private void visit(HSQLDBSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java new file mode 100644 index 000000000..d4126bd6e --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java @@ -0,0 +1,229 @@ +package sqlancer.hsqldb.ast; +import sqlancer.common.ast.newast.Node; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public class HSQLDBConstant implements Node { + + private HSQLDBConstant() { + } + public static class HSQLDBNullConstant extends HSQLDBConstant { + + @Override + public String toString() { + return "Null"; + } + + } + + public static class HSQLDBIntConstant extends HSQLDBConstant { + + private final int value; + + public HSQLDBIntConstant(long value) { + this.value = (int)value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class HSQLDBDoubleConstant extends HSQLDBConstant { + + private final double value; + + public HSQLDBDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "1.0e1/0.0e1"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-1.0e1/0.0e1"; + } + return String.valueOf(value); + } + + } + + public static class HSQLDBTextConstant extends HSQLDBConstant { + + private final String value; + + public HSQLDBTextConstant(String value) { + this.value = value; + } + + public HSQLDBTextConstant(String value, int size) { + this.value = value.substring(0, Math.min(value.length(), size)); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + + public static class HSQLDBBinaryConstant extends HSQLDBConstant { + + private final String value; + + public HSQLDBBinaryConstant(long value, int size) { + StringBuilder hex = new StringBuilder(Long.toHexString(value)); + if(hex.length() < 2) { + hex.append('0'); + } + this.value = hex.substring(0, Math.min(hex.length(), size)); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "X'" + value + "'"; + } + + } + + public static class HSQLDBDateConstant extends HSQLDBConstant { + + public String textRepr; + + public HSQLDBDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + } + + public static class HSQLDBTimestampConstant extends HSQLDBConstant { + + public String textRepr; + + public HSQLDBTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + } + + public static class HSQLDBTimeConstant extends HSQLDBConstant { + + public String textRepr; + + public HSQLDBTimeConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIME '%s'", textRepr); + } + + } + + public static class HSQLDBBooleanConstant extends HSQLDBConstant { + + private final boolean value; + + public HSQLDBBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static Node createStringConstant(String text, int size) { + return new HSQLDBTextConstant(text, size); + } + + public static Node createFloatConstant(double val) { + return new HSQLDBDoubleConstant(val); + } + + public static Node createIntConstant(long val) { + return new HSQLDBIntConstant(val); + } + + public static Node createNullConstant() { + return new HSQLDBNullConstant(); + } + + public static Node createBooleanConstant(boolean val) { + return new HSQLDBBooleanConstant(val); + } + + public static Node createDateConstant(long integer) { + return new HSQLDBDateConstant(integer); + } + + public static Node createTimeConstant(long integer, int size) { + return new HSQLDBTimeConstant(integer); + } + + public static Node createTimestampConstant(long integer, int size) { + return new HSQLDBTimestampConstant(integer); + } + + public static Node createBinaryConstant(long nonCachedInteger, int size) { + return new HSQLDBBinaryConstant(nonCachedInteger, size); + } + + +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBExpression.java b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java new file mode 100644 index 000000000..b5359c11c --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java @@ -0,0 +1,4 @@ +package sqlancer.hsqldb.ast; + +public interface HSQLDBExpression { +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java new file mode 100644 index 000000000..524702839 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java @@ -0,0 +1,124 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; + +import java.util.ArrayList; +import java.util.List; + +public class HSQLDBJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + private OuterType outerType; + + public enum JoinType { + INNER, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum OuterType { + FULL, LEFT, RIGHT; + + public static OuterType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public HSQLDBJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + private void setOuterType(OuterType outerType) { + this.outerType = outerType; + } + + public OuterType getOuterType() { + return outerType; + } + + public static List> getJoins( + List> tableList, HSQLDBGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(globalState).setColumns(columns); + switch (HSQLDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case NATURAL: + joinExpressions.add(HSQLDBJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); + break; + case LEFT: + joinExpressions + .add(HSQLDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case RIGHT: + joinExpressions + .add(HSQLDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static HSQLDBJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new HSQLDBJoin(left, right, JoinType.RIGHT, predicate); + } + + public static HSQLDBJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new HSQLDBJoin(left, right, JoinType.LEFT, predicate); + } + + public static HSQLDBJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new HSQLDBJoin(left, right, JoinType.INNER, predicate); + } + + public static Node createNaturalJoin(TableReferenceNode left, + TableReferenceNode right, OuterType naturalJoinType) { + HSQLDBJoin join = new HSQLDBJoin(left, right, JoinType.NATURAL, null); + join.setOuterType(naturalJoinType); + return join; + } + +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java new file mode 100644 index 000000000..05cd390f2 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java @@ -0,0 +1,18 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class HSQLDBSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java new file mode 100644 index 000000000..d01454a7f --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -0,0 +1,88 @@ +package sqlancer.hsqldb.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.ast.HSQLDBConstant; +import sqlancer.hsqldb.ast.HSQLDBExpression; + + +public final class HSQLDBExpressionGenerator extends TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { + + HSQLDBProvider.HSQLDBGlobalState hsqldbGlobalState; + public HSQLDBExpressionGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { + this.hsqldbGlobalState = globalState; + } + + @Override + public Node generatePredicate() { + return null; + } + + @Override + public Node negatePredicate(Node predicate) { + return null; + } + + @Override + public Node isNull(Node expr) { + return null; + } + + @Override + public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataType type) { + if (type.getType() == HSQLDBSchema.HSQLDBDataType.NULL || Randomly.getBooleanWithSmallProbability()) { + return HSQLDBConstant.createNullConstant(); + } + switch (type.getType()) { + case CHAR: + return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getAlphabeticChar(), type.getSize()); + case VARCHAR: + return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getString(), type.getSize()); + case TIME: + return HSQLDBConstant.createTimeConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); + case TIMESTAMP: + return HSQLDBConstant.createTimestampConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize() ); + + case INTEGER: + return HSQLDBConstant.HSQLDBIntConstant.createIntConstant(Randomly.getNonCachedInteger()); + case DOUBLE: + return HSQLDBConstant.HSQLDBDoubleConstant.createFloatConstant(hsqldbGlobalState.getRandomly().getDouble()); + case BOOLEAN: + return HSQLDBConstant.HSQLDBBooleanConstant.createBooleanConstant(Randomly.getBoolean()); + case DATE: + return HSQLDBConstant.createDateConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis())); + case BINARY: + return HSQLDBConstant.createBinaryConstant(Randomly.getNonCachedInteger(), type.getSize()); +// case OTHER: +// break; +// case NULL: +// break; + default: + throw new AssertionError("Unknown type: " + type); + } + } + + @Override + protected Node generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { + return null; + } + + @Override + protected Node generateColumn(HSQLDBSchema.HSQLDBCompositeDataType type) { + return null; + } + + @Override + protected HSQLDBSchema.HSQLDBCompositeDataType getRandomType() { + return HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull(); + } + + @Override + protected boolean canGenerateColumnOfType(HSQLDBSchema.HSQLDBCompositeDataType type) { + return columns.stream().anyMatch(c -> c.getType() == type); + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java new file mode 100644 index 000000000..0ce3fcea9 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.hsqldb.gen; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hsqldb.HSQLDBErrors; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBExpression; + +import java.util.List; +import java.util.stream.Collectors; + +public class HSQLDBInsertGenerator extends AbstractInsertGenerator { + + private final HSQLDBProvider.HSQLDBGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public HSQLDBInsertGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { + return new HSQLDBInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + HSQLDBErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertValue(HSQLDBSchema.HSQLDBColumn column) { + Node expression = new HSQLDBExpressionGenerator(globalState).generateConstant(column.getType()); + String s = HSQLDBToStringVisitor.asString(expression); + sb.append(s); + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java new file mode 100644 index 000000000..58b1cc426 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java @@ -0,0 +1,57 @@ +package sqlancer.hsqldb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; + +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.List; + +public class HSQLDBTableGenerator { + + public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @Nullable String tableName) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String name = tableName; + if (tableName == null) { + name = globalState.getSchema().getFreeTableName(); + } + sb.append("CREATE TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(name); + sb.append("("); + List columns = getNewColumns(name); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + sb.append(columns.get(i).getType().getType().name()); + if(columns.get(i).getType().getSize() > 0) { + //Cannot specify size for non composite data types + sb.append("("); + sb.append(columns.get(i).getType().getSize()); + sb.append(")"); + } + } + sb.append(")"); + sb.append(";"); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private static List getNewColumns(String tableName) { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + HSQLDBSchema.HSQLDBCompositeDataType columnType = HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull(); + columns.add(new HSQLDBSchema.HSQLDBColumn(columnName, null, columnType)); + } + return columns; + } +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java new file mode 100644 index 000000000..92e751300 --- /dev/null +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -0,0 +1,42 @@ +package sqlancer.hsqldb.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hsqldb.HSQLDBErrors; +import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBSchema; + +import java.util.List; + +public final class HSQLDBUpdateGenerator { + + private HSQLDBUpdateGenerator() { + } + + public static SQLQueryAdapter getQuery(HSQLDBGlobalState globalState) { + StringBuilder sb = new StringBuilder("UPDATE "); + ExpectedErrors errors = new ExpectedErrors(); + HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + //HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(" SET "); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append("="); +// Node expr; +// if (Randomly.getBooleanWithSmallProbability()) { +// //expr = gen.generateExpression(); +// HSQLDBErrors.addExpressionErrors(errors); +// } else { +// //expr = gen.generateConstant(); +// } + } + HSQLDBErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java new file mode 100644 index 000000000..69216dca7 --- /dev/null +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -0,0 +1,25 @@ +package sqlancer.hsqldb.test; + +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.ast.HSQLDBExpression; + +public class HSQLDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> implements TestOracle { + + public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { + super(state); + } + + @Override + protected ExpressionGenerator> getGen() { + return null; + } + + @Override + public void check() throws Exception { + + } +} From 94693f7e92cf5bf7128897c1b174b7bd39d7d88d Mon Sep 17 00:00:00 2001 From: Nishita Date: Fri, 25 Nov 2022 18:46:29 +0530 Subject: [PATCH 240/856] Added Update statements --- src/sqlancer/hsqldb/HSQLDBProvider.java | 2 +- .../hsqldb/gen/HSQLDBInsertGenerator.java | 3 +- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 30 ++++++++++++------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/hsqldb/HSQLDBProvider.java b/src/sqlancer/hsqldb/HSQLDBProvider.java index cd238df23..351c8ab8c 100644 --- a/src/sqlancer/hsqldb/HSQLDBProvider.java +++ b/src/sqlancer/hsqldb/HSQLDBProvider.java @@ -85,7 +85,7 @@ private static int mapActions(HSQLDBProvider.HSQLDBGlobalState globalState, HSQL case INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case UPDATE: - return r.getInteger(0, 1); + return r.getInteger(0, 10); default: throw new AssertionError(a); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index 0ce3fcea9..e53b67d44 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -3,7 +3,6 @@ import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.hsqldb.HSQLDBErrors; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.HSQLDBToStringVisitor; @@ -35,7 +34,7 @@ private SQLQueryAdapter generate() { sb.append(")"); sb.append(" VALUES "); insertColumns(columns); - HSQLDBErrors.addInsertErrors(errors); + //HSQLDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index 92e751300..97b8f6e18 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -1,24 +1,32 @@ package sqlancer.hsqldb.gen; +import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.hsqldb.HSQLDBErrors; -import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBExpression; import java.util.List; public final class HSQLDBUpdateGenerator { - private HSQLDBUpdateGenerator() { + private final static ExpectedErrors errors = new ExpectedErrors(); + private final HSQLDBProvider.HSQLDBGlobalState globalState; + private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { + this.globalState = globalState; } - public static SQLQueryAdapter getQuery(HSQLDBGlobalState globalState) { + public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { + return new HSQLDBUpdateGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); - //HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); List columns = table.getRandomNonEmptyColumnSubset(); for (int i = 0; i < columns.size(); i++) { @@ -27,15 +35,15 @@ public static SQLQueryAdapter getQuery(HSQLDBGlobalState globalState) { } sb.append(columns.get(i).getName()); sb.append("="); -// Node expr; + Node expr; // if (Randomly.getBooleanWithSmallProbability()) { -// //expr = gen.generateExpression(); +// expr = gen.generateExpression(columns.get(i).getType()); // HSQLDBErrors.addExpressionErrors(errors); // } else { -// //expr = gen.generateConstant(); -// } + expr = gen.generateConstant(columns.get(i).getType()); + //} + sb.append(HSQLDBToStringVisitor.asString(expr)); } - HSQLDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } From 58508fed82d17b9de0cf1012056dc870069d86c0 Mon Sep 17 00:00:00 2001 From: Nishita Date: Mon, 28 Nov 2022 12:00:56 +0530 Subject: [PATCH 241/856] Fixes Checkstyle violations and adds support for expression --- src/sqlancer/hsqldb/HSQLDBErrors.java | 3 + src/sqlancer/hsqldb/HSQLDBOptions.java | 11 +- src/sqlancer/hsqldb/HSQLDBProvider.java | 54 +++++---- src/sqlancer/hsqldb/HSQLDBSchema.java | 78 +++++++------ .../hsqldb/HSQLDBToStringVisitor.java | 3 +- src/sqlancer/hsqldb/ast/HSQLDBConstant.java | 10 +- src/sqlancer/hsqldb/ast/HSQLDBJoin.java | 51 ++++----- .../ast/HSQLDBUnaryPostfixOperation.java | 78 +++++++++++++ .../ast/HSQLDBUnaryPrefixOperation.java | 83 ++++++++++++++ .../hsqldb/gen/HSQLDBExpressionGenerator.java | 103 +++++++++++++----- .../hsqldb/gen/HSQLDBInsertGenerator.java | 12 +- .../hsqldb/gen/HSQLDBTableGenerator.java | 19 ++-- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 21 ++-- .../test/HSQLDBQueryPartitioningBase.java | 49 ++++++++- .../HSQLDBQueryPartitioningWhereTester.java | 43 ++++++++ 15 files changed, 465 insertions(+), 153 deletions(-) create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java create mode 100644 src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java diff --git a/src/sqlancer/hsqldb/HSQLDBErrors.java b/src/sqlancer/hsqldb/HSQLDBErrors.java index 5fa77051a..8e2a8cd2f 100644 --- a/src/sqlancer/hsqldb/HSQLDBErrors.java +++ b/src/sqlancer/hsqldb/HSQLDBErrors.java @@ -8,12 +8,15 @@ private HSQLDBErrors() { } public static void addExpressionErrors(ExpectedErrors errors) { + errors.add(""); } private static void addRegexErrors(ExpectedErrors errors) { + errors.add(""); } private static void addFunctionErrors(ExpectedErrors errors) { + errors.add(""); } public static void addInsertErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index b2418fe8d..54954b797 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -1,13 +1,14 @@ package sqlancer.hsqldb; +import java.sql.SQLException; +import java.util.List; + import com.beust.jcommander.Parameter; + import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; -import sqlancer.hsqldb.test.HSQLDBQueryPartitioningBase; - -import java.sql.SQLException; -import java.util.List; +import sqlancer.hsqldb.test.HSQLDBQueryPartitioningWhereTester; public class HSQLDBOptions implements DBMSSpecificOptions { @@ -18,7 +19,7 @@ public enum HSQLDBOracleFactory implements OracleFactory { @@ -22,9 +31,8 @@ public HSQLDBProvider() { super(HSQLDBGlobalState.class, HSQLDBOptions.class); } - public enum Action implements AbstractAction { - INSERT(HSQLDBInsertGenerator::getQuery), - UPDATE(HSQLDBUpdateGenerator::getQuery); + public enum Action implements AbstractAction { + INSERT(HSQLDBInsertGenerator::getQuery), UPDATE(HSQLDBUpdateGenerator::getQuery); private final SQLQueryProvider sqlQueryProvider; @@ -44,7 +52,7 @@ public SQLConnection createDatabase(HSQLDBGlobalState globalState) throws Except String url = "jdbc:hsqldb:file:" + databaseName; MainOptions options = globalState.getOptions(); Connection connection = DriverManager.getConnection(url, options.getUserName(), options.getPassword()); - //When a server instance is started, or when a connection is made to an in-process database, + // When a server instance is started, or when a connection is made to an in-process database, // a new, empty database is created if no database exists at the given path. try (Statement s = connection.createStatement()) { s.execute("DROP SCHEMA PUBLIC CASCADE"); @@ -70,24 +78,24 @@ public void generateDatabase(HSQLDBGlobalState globalState) throws Exception { if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } - StatementExecutor se = new StatementExecutor<>(globalState, HSQLDBProvider.Action.values(), - HSQLDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); + StatementExecutor se = new StatementExecutor<>(globalState, + HSQLDBProvider.Action.values(), HSQLDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); se.executeStatements(); } private static int mapActions(HSQLDBProvider.HSQLDBGlobalState globalState, HSQLDBProvider.Action a) { Randomly r = globalState.getRandomly(); switch (a) { - case INSERT: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case UPDATE: - return r.getInteger(0, 10); - default: - throw new AssertionError(a); + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case UPDATE: + return r.getInteger(0, 10); + default: + throw new AssertionError(a); } } @@ -100,5 +108,3 @@ protected HSQLDBSchema readSchema() throws SQLException { } } - - diff --git a/src/sqlancer/hsqldb/HSQLDBSchema.java b/src/sqlancer/hsqldb/HSQLDBSchema.java index 13059ef3e..e1e1cb94b 100644 --- a/src/sqlancer/hsqldb/HSQLDBSchema.java +++ b/src/sqlancer/hsqldb/HSQLDBSchema.java @@ -1,5 +1,12 @@ package sqlancer.hsqldb; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.DBMSCommon; @@ -8,20 +15,13 @@ import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.TableIndex; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - public class HSQLDBSchema extends AbstractSchema { public HSQLDBSchema(List databaseTables) { super(databaseTables); } - public static HSQLDBSchema fromConnection(SQLConnection connection, String databaseName) throws SQLException { + public static HSQLDBSchema fromConnection(SQLConnection connection, String databaseName) throws SQLException { List databaseTables = new ArrayList<>(); List tableNames = getTableNames(connection); for (String tableName : tableNames) { @@ -43,7 +43,8 @@ public static HSQLDBSchema fromConnection(SQLConnection connection, String datab private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'PUBLIC';")) { + try (ResultSet rs = s + .executeQuery("SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'PUBLIC';")) { while (rs.next()) { tableNames.add(rs.getString("TABLE_NAME")); } @@ -59,8 +60,9 @@ private static List getTableColumns(SQLConnection con, String tabl try (ResultSet rs = s.executeQuery(String.format(sql, tableName))) { while (rs.next()) { HSQLDBDataType dataType = HSQLDBDataType.from(rs.getString("TYPE_NAME")); - HSQLDBCompositeDataType compositeDataType = new HSQLDBCompositeDataType(dataType, rs.getInt("COLUMN_SIZE")); - HSQLDBColumn column = new HSQLDBColumn(rs.getString("COLUMN_NAME"), null, compositeDataType ); + HSQLDBCompositeDataType compositeDataType = new HSQLDBCompositeDataType(dataType, + rs.getInt("COLUMN_SIZE")); + HSQLDBColumn column = new HSQLDBColumn(rs.getString("COLUMN_NAME"), null, compositeDataType); tableNames.add(column); } } @@ -68,7 +70,8 @@ private static List getTableColumns(SQLConnection con, String tabl return tableNames; } - public static class HSQLDBTable extends AbstractRelationalTable { + public static class HSQLDBTable + extends AbstractRelationalTable { public HSQLDBTable(String tableName, List columns, boolean isView) { super(tableName, columns, Collections.emptyList(), isView); @@ -76,8 +79,8 @@ public HSQLDBTable(String tableName, List columns, bo } - public static class HSQLDBColumn extends AbstractTableColumn - { + public static class HSQLDBColumn + extends AbstractTableColumn { public HSQLDBColumn(String name, HSQLDBTable table, HSQLDBCompositeDataType type) { super(name, table, type); @@ -96,9 +99,9 @@ public static HSQLDBSchema.HSQLDBDataType getRandomWithoutNull() { return dt; } - public static HSQLDBDataType from(String type_name) { - for( HSQLDBDataType value : HSQLDBDataType.values()) { - if(value.name().equals(type_name)) { + public static HSQLDBDataType from(String typeName) { + for (HSQLDBDataType value : HSQLDBDataType.values()) { + if (value.name().equals(typeName)) { return value; } } @@ -106,7 +109,7 @@ public static HSQLDBDataType from(String type_name) { } } - public static class HSQLDBCompositeDataType{ + public static class HSQLDBCompositeDataType { private final int size; private final HSQLDBDataType type; @@ -115,28 +118,31 @@ public HSQLDBCompositeDataType(HSQLDBDataType type, int size) { this.size = size; } - public static HSQLDBCompositeDataType getRandomWithoutNull() { HSQLDBSchema.HSQLDBDataType type = HSQLDBSchema.HSQLDBDataType.getRandomWithoutNull(); + return getRandomWithType(type); + } + + public static HSQLDBCompositeDataType getRandomWithType(HSQLDBSchema.HSQLDBDataType type) { int size; switch (type) { - case VARCHAR: - case CHAR: - case TIME: - case BINARY: - case TIMESTAMP: - size = Randomly.fromOptions(4,6,8); - break; - case BOOLEAN: - case INTEGER: - case DOUBLE: - //case UUID: - //case OTHER: - case DATE: - size = 0; - break; - default: - throw new AssertionError(type); + case VARCHAR: + case CHAR: + case TIME: + case BINARY: + case TIMESTAMP: + size = Randomly.fromOptions(4, 6, 8); + break; + case BOOLEAN: + case INTEGER: + case DOUBLE: + // case UUID: + // case OTHER: + case DATE: + size = 0; + break; + default: + throw new AssertionError(type); } return new HSQLDBSchema.HSQLDBCompositeDataType(type, size); diff --git a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java index 66ccbfd12..722e241fa 100644 --- a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java +++ b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java @@ -9,7 +9,6 @@ public class HSQLDBToStringVisitor extends NewToStringVisitor { - @Override public void visitSpecific(Node expr) { if (expr instanceof HSQLDBConstant) { @@ -44,7 +43,7 @@ private void visit(HSQLDBJoin join) { visit(join.getOnCondition()); } } - + private void visit(HSQLDBConstant constant) { sb.append(constant.toString()); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java index d4126bd6e..f6eed5e13 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java @@ -1,13 +1,15 @@ package sqlancer.hsqldb.ast; -import sqlancer.common.ast.newast.Node; import java.sql.Timestamp; import java.text.SimpleDateFormat; +import sqlancer.common.ast.newast.Node; + public class HSQLDBConstant implements Node { private HSQLDBConstant() { } + public static class HSQLDBNullConstant extends HSQLDBConstant { @Override @@ -22,7 +24,7 @@ public static class HSQLDBIntConstant extends HSQLDBConstant { private final int value; public HSQLDBIntConstant(long value) { - this.value = (int)value; + this.value = (int) value; } @Override @@ -83,14 +85,13 @@ public String toString() { } - public static class HSQLDBBinaryConstant extends HSQLDBConstant { private final String value; public HSQLDBBinaryConstant(long value, int size) { StringBuilder hex = new StringBuilder(Long.toHexString(value)); - if(hex.length() < 2) { + if (hex.length() < 2) { hex.append('0'); } this.value = hex.substring(0, Math.min(hex.length(), size)); @@ -225,5 +226,4 @@ public static Node createBinaryConstant(long nonCachedInteger, return new HSQLDBBinaryConstant(nonCachedInteger, size); } - } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java index 524702839..ef715052d 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java @@ -1,5 +1,8 @@ package sqlancer.hsqldb.ast; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; @@ -8,9 +11,6 @@ import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -import java.util.ArrayList; -import java.util.List; - public class HSQLDBJoin implements Node { private final TableReferenceNode leftTable; @@ -36,8 +36,8 @@ public static OuterType getRandom() { } public HSQLDBJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; @@ -78,39 +78,40 @@ public static List> getJoins( columns.addAll(rightTable.getTable().getColumns()); HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(globalState).setColumns(columns); switch (HSQLDBJoin.JoinType.getRandom()) { - case INNER: - joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - case NATURAL: - joinExpressions.add(HSQLDBJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); - break; - case LEFT: - joinExpressions - .add(HSQLDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - case RIGHT: - joinExpressions - .add(HSQLDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - default: - throw new AssertionError(); - } + case INNER: + joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTable, rightTable, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case NATURAL: + joinExpressions.add(HSQLDBJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); + break; + case LEFT: + joinExpressions.add(HSQLDBJoin.createLeftOuterJoin(leftTable, rightTable, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case RIGHT: + joinExpressions.add(HSQLDBJoin.createRightOuterJoin(leftTable, rightTable, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + default: + throw new AssertionError(); + } } return joinExpressions; } public static HSQLDBJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new HSQLDBJoin(left, right, JoinType.RIGHT, predicate); } public static HSQLDBJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new HSQLDBJoin(left, right, JoinType.LEFT, predicate); } public static HSQLDBJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + TableReferenceNode right, Node predicate) { return new HSQLDBJoin(left, right, JoinType.INNER, predicate); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java new file mode 100644 index 000000000..12f8137e0 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java @@ -0,0 +1,78 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { + + // private final Node expr; + // private final HSQLDBUnaryPostfixOperator op; + private boolean negate; + + public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op, boolean negate) { + super(expr, op); + // this.expr = expr; + // this.op = op; + setNegate(negate); + } + + public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op) { + super(expr, op); + // this.expr = expr; + // this.op = op; + } + + public enum HSQLDBUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public HSQLDBSchema.HSQLDBDataType[] getInputDataTypes() { + return HSQLDBSchema.HSQLDBDataType.values(); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public HSQLDBSchema.HSQLDBDataType[] getInputDataTypes() { + return HSQLDBSchema.HSQLDBDataType.values(); + } + }; + + private final String textRepresentations; + + HSQLDBUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static HSQLDBUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract HSQLDBSchema.HSQLDBDataType[] getInputDataTypes(); + + } + + public boolean isNegated() { + return negate; + } + + public void setNegate(boolean negate) { + this.negate = negate; + } + + public Node getExpression() { + return getExpr(); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java new file mode 100644 index 000000000..233b9e943 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java @@ -0,0 +1,83 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { + + // private final HSQLDBUnaryPrefixOperation.HSQLDBUnaryPrefixOperator operation; + // private final Node expression; + + public HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperator operation, Node expression) { + super(expression, operation); + // this.operation = operation; + // this.expression = expression; + } + + @Override + public String getOperatorRepresentation() { + return null; + } + + public enum HSQLDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", HSQLDBSchema.HSQLDBDataType.BOOLEAN, HSQLDBSchema.HSQLDBDataType.INTEGER) { + @Override + public HSQLDBSchema.HSQLDBDataType getExpressionType() { + return HSQLDBSchema.HSQLDBDataType.BOOLEAN; + } + + @Override + protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { + return null; // TODO + } + }, + + UNARY_PLUS("+", HSQLDBSchema.HSQLDBDataType.INTEGER) { + @Override + public HSQLDBSchema.HSQLDBDataType getExpressionType() { + return HSQLDBSchema.HSQLDBDataType.INTEGER; + } + + @Override + protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { + return expectedValue; + } + }, + UNARY_MINUS("-", HSQLDBSchema.HSQLDBDataType.INTEGER) { + @Override + public HSQLDBSchema.HSQLDBDataType getExpressionType() { + return HSQLDBSchema.HSQLDBDataType.INTEGER; + } + + @Override + protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { + return null; + } + }; + + private String textRepresentation; + private HSQLDBSchema.HSQLDBDataType[] dataTypes; + + HSQLDBUnaryPrefixOperator(String textRepresentation, HSQLDBSchema.HSQLDBDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract HSQLDBSchema.HSQLDBDataType getExpressionType(); + + public HSQLDBSchema.HSQLDBDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); + } + + protected abstract HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + } + +} diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index d01454a7f..6e9be4fcf 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -1,34 +1,48 @@ package sqlancer.hsqldb.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBUnaryPostfixOperation; +import sqlancer.hsqldb.ast.HSQLDBUnaryPrefixOperation; +public final class HSQLDBExpressionGenerator extends + TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { -public final class HSQLDBExpressionGenerator extends TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { + private enum Expression { + BINARY_LOGICAL; + } HSQLDBProvider.HSQLDBGlobalState hsqldbGlobalState; + public HSQLDBExpressionGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { this.hsqldbGlobalState = globalState; } @Override public Node generatePredicate() { - return null; + return generateExpression( + HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithType(HSQLDBSchema.HSQLDBDataType.BOOLEAN)); } @Override public Node negatePredicate(Node predicate) { - return null; + return new HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperation.HSQLDBUnaryPrefixOperator.NOT, predicate); } @Override public Node isNull(Node expr) { - return null; + return new HSQLDBUnaryPostfixOperation(expr, HSQLDBUnaryPostfixOperation.HSQLDBUnaryPostfixOperator.IS_NULL); } @Override @@ -37,37 +51,52 @@ public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataT return HSQLDBConstant.createNullConstant(); } switch (type.getType()) { - case CHAR: - return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getAlphabeticChar(), type.getSize()); - case VARCHAR: - return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getString(), type.getSize()); - case TIME: - return HSQLDBConstant.createTimeConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); - case TIMESTAMP: - return HSQLDBConstant.createTimestampConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize() ); - - case INTEGER: - return HSQLDBConstant.HSQLDBIntConstant.createIntConstant(Randomly.getNonCachedInteger()); - case DOUBLE: - return HSQLDBConstant.HSQLDBDoubleConstant.createFloatConstant(hsqldbGlobalState.getRandomly().getDouble()); - case BOOLEAN: - return HSQLDBConstant.HSQLDBBooleanConstant.createBooleanConstant(Randomly.getBoolean()); - case DATE: - return HSQLDBConstant.createDateConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis())); - case BINARY: - return HSQLDBConstant.createBinaryConstant(Randomly.getNonCachedInteger(), type.getSize()); -// case OTHER: -// break; -// case NULL: -// break; - default: - throw new AssertionError("Unknown type: " + type); + case CHAR: + return HSQLDBConstant.HSQLDBTextConstant + .createStringConstant(hsqldbGlobalState.getRandomly().getAlphabeticChar(), type.getSize()); + case VARCHAR: + return HSQLDBConstant.HSQLDBTextConstant.createStringConstant(hsqldbGlobalState.getRandomly().getString(), + type.getSize()); + case TIME: + return HSQLDBConstant.createTimeConstant( + hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); + case TIMESTAMP: + return HSQLDBConstant.createTimestampConstant( + hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis()), type.getSize()); + + case INTEGER: + return HSQLDBConstant.HSQLDBIntConstant.createIntConstant(Randomly.getNonCachedInteger()); + case DOUBLE: + return HSQLDBConstant.HSQLDBDoubleConstant.createFloatConstant(hsqldbGlobalState.getRandomly().getDouble()); + case BOOLEAN: + return HSQLDBConstant.HSQLDBBooleanConstant.createBooleanConstant(Randomly.getBoolean()); + case DATE: + return HSQLDBConstant + .createDateConstant(hsqldbGlobalState.getRandomly().getLong(0, System.currentTimeMillis())); + case BINARY: + return HSQLDBConstant.createBinaryConstant(Randomly.getNonCachedInteger(), type.getSize()); + default: + throw new AssertionError("Unknown type: " + type); } } @Override protected Node generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { - return null; + if (depth >= hsqldbGlobalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } + + List possibleOptions = new ArrayList<>( + Arrays.asList(HSQLDBExpressionGenerator.Expression.values())); + + HSQLDBExpressionGenerator.Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case BINARY_LOGICAL: + BinaryOperatorNode.Operator op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(type), generateExpression(type), op); + default: + throw new AssertionError(); + } } @Override @@ -85,4 +114,18 @@ protected boolean canGenerateColumnOfType(HSQLDBSchema.HSQLDBCompositeDataType t return columns.stream().anyMatch(c -> c.getType() == type); } + public enum HSQLDBBinaryLogicalOperator implements BinaryOperatorNode.Operator { + + AND, OR; + + @Override + public String getTextRepresentation() { + return toString(); + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + } } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index e53b67d44..c21d7c786 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -1,4 +1,8 @@ package sqlancer.hsqldb.gen; + +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; @@ -8,9 +12,6 @@ import sqlancer.hsqldb.HSQLDBToStringVisitor; import sqlancer.hsqldb.ast.HSQLDBExpression; -import java.util.List; -import java.util.stream.Collectors; - public class HSQLDBInsertGenerator extends AbstractInsertGenerator { private final HSQLDBProvider.HSQLDBGlobalState globalState; @@ -34,13 +35,14 @@ private SQLQueryAdapter generate() { sb.append(")"); sb.append(" VALUES "); insertColumns(columns); - //HSQLDBErrors.addInsertErrors(errors); + // HSQLDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } @Override protected void insertValue(HSQLDBSchema.HSQLDBColumn column) { - Node expression = new HSQLDBExpressionGenerator(globalState).generateConstant(column.getType()); + Node expression = new HSQLDBExpressionGenerator(globalState) + .generateConstant(column.getType()); String s = HSQLDBToStringVisitor.asString(expression); sb.append(s); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java index 58b1cc426..48606e9bf 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java @@ -1,15 +1,15 @@ package sqlancer.hsqldb.gen; +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; + import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.List; - public class HSQLDBTableGenerator { public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @Nullable String tableName) { @@ -25,7 +25,7 @@ public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @N } sb.append(name); sb.append("("); - List columns = getNewColumns(name); + List columns = getNewColumns(); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); @@ -33,8 +33,8 @@ public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @N sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType().getType().name()); - if(columns.get(i).getType().getSize() > 0) { - //Cannot specify size for non composite data types + if (columns.get(i).getType().getSize() > 0) { + // Cannot specify size for non composite data types sb.append("("); sb.append(columns.get(i).getType().getSize()); sb.append(")"); @@ -45,11 +45,12 @@ public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @N return new SQLQueryAdapter(sb.toString(), errors, true); } - private static List getNewColumns(String tableName) { + private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { String columnName = String.format("c%d", i); - HSQLDBSchema.HSQLDBCompositeDataType columnType = HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull(); + HSQLDBSchema.HSQLDBCompositeDataType columnType = HSQLDBSchema.HSQLDBCompositeDataType + .getRandomWithoutNull(); columns.add(new HSQLDBSchema.HSQLDBColumn(columnName, null, columnType)); } return columns; diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index 97b8f6e18..7963aca90 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -1,5 +1,7 @@ package sqlancer.hsqldb.gen; +import java.util.List; + import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -8,12 +10,11 @@ import sqlancer.hsqldb.HSQLDBToStringVisitor; import sqlancer.hsqldb.ast.HSQLDBExpression; -import java.util.List; - public final class HSQLDBUpdateGenerator { - private final static ExpectedErrors errors = new ExpectedErrors(); + private static final ExpectedErrors EXPECTED_ERRORS = new ExpectedErrors(); private final HSQLDBProvider.HSQLDBGlobalState globalState; + private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { this.globalState = globalState; } @@ -36,15 +37,15 @@ private SQLQueryAdapter getQuery() { sb.append(columns.get(i).getName()); sb.append("="); Node expr; -// if (Randomly.getBooleanWithSmallProbability()) { -// expr = gen.generateExpression(columns.get(i).getType()); -// HSQLDBErrors.addExpressionErrors(errors); -// } else { - expr = gen.generateConstant(columns.get(i).getType()); - //} + // if (Randomly.getBooleanWithSmallProbability()) { + // expr = gen.generateExpression(columns.get(i).getType()); + // HSQLDBErrors.addExpressionErrors(errors); + // } else { + expr = gen.generateConstant(columns.get(i).getType()); + // } sb.append(HSQLDBToStringVisitor.asString(expr)); } - return new SQLQueryAdapter(sb.toString(), errors); + return new SQLQueryAdapter(sb.toString(), EXPECTED_ERRORS); } } diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java index 69216dca7..d6d3d553e 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -1,13 +1,33 @@ package sqlancer.hsqldb.test; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; + +public class HSQLDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> + implements TestOracle { + + HSQLDBSelect select; + HSQLDBExpressionGenerator expressionGenerator; -public class HSQLDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> implements TestOracle { + HSQLDBSchema schema; + + List targetTables; public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { super(state); @@ -15,11 +35,36 @@ public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { @Override protected ExpressionGenerator> getGen() { - return null; + return expressionGenerator; } @Override public void check() throws Exception { + schema = state.getSchema(); + targetTables = schema.getDatabaseTablesRandomSubsetNotEmpty(); + expressionGenerator = new HSQLDBExpressionGenerator(state) + .setColumns(targetTables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList())); + initializeTernaryPredicateVariants(); + select = new HSQLDBSelect(); + select.setFetchColumns(generateFetchColumns()); + List> tableList = targetTables.stream() + .map(t -> new TableReferenceNode(t)) + .collect(Collectors.toList()); + List> joins = HSQLDBJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new HSQLDBSchema.HSQLDBColumn("*", null, null))); + } else { + columns = Randomly.nonEmptySubset(targetTables.stream().flatMap(t -> t.getColumns().stream()) + .map(c -> new ColumnReferenceNode(c)) + .collect(Collectors.toList())); + } + return columns; } } diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java new file mode 100644 index 000000000..09f51051d --- /dev/null +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java @@ -0,0 +1,43 @@ +package sqlancer.hsqldb.test; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.duckdb.test.DuckDBQueryPartitioningBase; +import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBToStringVisitor; + +public class HSQLDBQueryPartitioningWhereTester extends HSQLDBQueryPartitioningBase { + + public HSQLDBQueryPartitioningWhereTester(HSQLDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + select.setWhereClause(null); + String originalQueryString = HSQLDBToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByExpressions(expressionGenerator.generateOrderBys()); + } + select.setWhereClause(predicate); + String firstQueryString = HSQLDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = HSQLDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = HSQLDBToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + } + +} From e68539d42b68e400cfc5bfc4d972e8d948aee531 Mon Sep 17 00:00:00 2001 From: Nishita Date: Mon, 28 Nov 2022 18:30:53 +0530 Subject: [PATCH 242/856] Adds Binary Comparsion and Binary arithmetic expressions --- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 58 ++++++++++++++++++- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index 6e9be4fcf..ec8a26b76 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -20,7 +20,7 @@ public final class HSQLDBExpressionGenerator extends TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { private enum Expression { - BINARY_LOGICAL; + BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC; } HSQLDBProvider.HSQLDBGlobalState hsqldbGlobalState; @@ -90,13 +90,23 @@ protected Node generateExpression(HSQLDBSchema.HSQLDBComposite Arrays.asList(HSQLDBExpressionGenerator.Expression.values())); HSQLDBExpressionGenerator.Expression expr = Randomly.fromList(possibleOptions); + BinaryOperatorNode.Operator op; switch (expr) { case BINARY_LOGICAL: - BinaryOperatorNode.Operator op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(type), generateExpression(type), op); + op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); + break; + case BINARY_COMPARISON: + op = HSQLDBDBBinaryComparisonOperator.getRandom(); + break; + case BINARY_ARITHMETIC: + op = HSQLDBDBBinaryArithmeticOperator.getRandom(); + break; default: throw new AssertionError(); } + + return new NewBinaryOperatorNode<>(generateExpression(type), generateExpression(type), op); + } @Override @@ -128,4 +138,46 @@ public static BinaryOperatorNode.Operator getRandom() { } } + + public enum HSQLDBDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), + REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); + + private String textRepr; + + HSQLDBDBBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + public enum HSQLDBDBBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), AND("&"), OR("|"), LSHIFT("<<"), RSHIFT(">>"); + + private String textRepr; + + HSQLDBDBBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } } From 0c037dda2be1150c1ae19e72d40cee92dea51517 Mon Sep 17 00:00:00 2001 From: Nishita Date: Mon, 28 Nov 2022 18:43:02 +0530 Subject: [PATCH 243/856] Fixes failing test --- src/sqlancer/hsqldb/HSQLDBOptions.java | 2 ++ src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index 54954b797..d025d4576 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -4,12 +4,14 @@ import java.util.List; import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; import sqlancer.hsqldb.test.HSQLDBQueryPartitioningWhereTester; +@Parameters(commandDescription = "hsqldb") public class HSQLDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index ec8a26b76..1db82eca9 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -94,8 +94,8 @@ protected Node generateExpression(HSQLDBSchema.HSQLDBComposite switch (expr) { case BINARY_LOGICAL: op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); - break; - case BINARY_COMPARISON: + break; + case BINARY_COMPARISON: op = HSQLDBDBBinaryComparisonOperator.getRandom(); break; case BINARY_ARITHMETIC: From b423e07335cf5d35a427b92508f7228890d5c507 Mon Sep 17 00:00:00 2001 From: Nishita Dutta Date: Tue, 29 Nov 2022 23:21:45 +0530 Subject: [PATCH 244/856] Expressions for SELECT (#1) Implements expressions for select clause in HSQLDB --- .../hsqldb/ast/HSQLDBColumnReference.java | 11 ++++++ .../ast/HSQLDBUnaryPostfixOperation.java | 21 ---------- .../ast/HSQLDBUnaryPrefixOperation.java | 31 +-------------- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 39 +++++++++++++------ .../hsqldb/gen/HSQLDBUpdateGenerator.java | 14 ++++--- .../test/HSQLDBQueryPartitioningBase.java | 11 ++++-- .../HSQLDBQueryPartitioningWhereTester.java | 4 +- 7 files changed, 56 insertions(+), 75 deletions(-) create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java diff --git a/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java new file mode 100644 index 000000000..b2179eca3 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBColumnReference extends ColumnReferenceNode { + + public HSQLDBColumnReference(HSQLDBSchema.HSQLDBColumn column) { + super(column); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java index 12f8137e0..9e63efee5 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java @@ -8,21 +8,8 @@ public class HSQLDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { - // private final Node expr; - // private final HSQLDBUnaryPostfixOperator op; - private boolean negate; - - public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op, boolean negate) { - super(expr, op); - // this.expr = expr; - // this.op = op; - setNegate(negate); - } - public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op) { super(expr, op); - // this.expr = expr; - // this.op = op; } public enum HSQLDBUnaryPostfixOperator implements BinaryOperatorNode.Operator { @@ -58,14 +45,6 @@ public String getTextRepresentation() { } - public boolean isNegated() { - return negate; - } - - public void setNegate(boolean negate) { - this.negate = negate; - } - public Node getExpression() { return getExpr(); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java index 233b9e943..b1327d0be 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java @@ -1,6 +1,5 @@ package sqlancer.hsqldb.ast; -import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.common.ast.newast.Node; @@ -8,18 +7,13 @@ public class HSQLDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { - // private final HSQLDBUnaryPrefixOperation.HSQLDBUnaryPrefixOperator operation; - // private final Node expression; - public HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperator operation, Node expression) { super(expression, operation); - // this.operation = operation; - // this.expression = expression; } @Override public String getOperatorRepresentation() { - return null; + return this.op.getTextRepresentation(); } public enum HSQLDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { @@ -28,11 +22,6 @@ public enum HSQLDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { public HSQLDBSchema.HSQLDBDataType getExpressionType() { return HSQLDBSchema.HSQLDBDataType.BOOLEAN; } - - @Override - protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { - return null; // TODO - } }, UNARY_PLUS("+", HSQLDBSchema.HSQLDBDataType.INTEGER) { @@ -40,40 +29,22 @@ protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { public HSQLDBSchema.HSQLDBDataType getExpressionType() { return HSQLDBSchema.HSQLDBDataType.INTEGER; } - - @Override - protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { - return expectedValue; - } }, UNARY_MINUS("-", HSQLDBSchema.HSQLDBDataType.INTEGER) { @Override public HSQLDBSchema.HSQLDBDataType getExpressionType() { return HSQLDBSchema.HSQLDBDataType.INTEGER; } - - @Override - protected HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue) { - return null; - } }; private String textRepresentation; - private HSQLDBSchema.HSQLDBDataType[] dataTypes; HSQLDBUnaryPrefixOperator(String textRepresentation, HSQLDBSchema.HSQLDBDataType... dataTypes) { this.textRepresentation = textRepresentation; - this.dataTypes = dataTypes.clone(); } public abstract HSQLDBSchema.HSQLDBDataType getExpressionType(); - public HSQLDBSchema.HSQLDBDataType getRandomInputDataTypes() { - return Randomly.fromOptions(dataTypes); - } - - protected abstract HSQLDBConstant getExpectedValue(HSQLDBConstant expectedValue); - @Override public String getTextRepresentation() { return this.textRepresentation; diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index 1db82eca9..a978dfdc3 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; @@ -11,6 +12,7 @@ import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; import sqlancer.hsqldb.ast.HSQLDBUnaryPostfixOperation; @@ -47,10 +49,9 @@ public Node isNull(Node expr) { @Override public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataType type) { - if (type.getType() == HSQLDBSchema.HSQLDBDataType.NULL || Randomly.getBooleanWithSmallProbability()) { - return HSQLDBConstant.createNullConstant(); - } switch (type.getType()) { + case NULL: + return HSQLDBConstant.createNullConstant(); case CHAR: return HSQLDBConstant.HSQLDBTextConstant .createStringConstant(hsqldbGlobalState.getRandomly().getAlphabeticChar(), type.getSize()); @@ -82,7 +83,8 @@ public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataT @Override protected Node generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { - if (depth >= hsqldbGlobalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + if (depth >= hsqldbGlobalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithSmallProbability()) { return generateLeafNode(type); } @@ -93,25 +95,26 @@ protected Node generateExpression(HSQLDBSchema.HSQLDBComposite BinaryOperatorNode.Operator op; switch (expr) { case BINARY_LOGICAL: + case BINARY_ARITHMETIC: op = HSQLDBExpressionGenerator.HSQLDBBinaryLogicalOperator.getRandom(); break; case BINARY_COMPARISON: op = HSQLDBDBBinaryComparisonOperator.getRandom(); break; - case BINARY_ARITHMETIC: - op = HSQLDBDBBinaryArithmeticOperator.getRandom(); - break; default: throw new AssertionError(); } - return new NewBinaryOperatorNode<>(generateExpression(type), generateExpression(type), op); + return new NewBinaryOperatorNode<>(generateExpression(type, depth + 1), generateExpression(type, depth + 1), + op); } @Override protected Node generateColumn(HSQLDBSchema.HSQLDBCompositeDataType type) { - return null; + HSQLDBSchema.HSQLDBColumn column = Randomly + .fromList(columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList())); + return new HSQLDBColumnReference(column); } @Override @@ -140,9 +143,7 @@ public static BinaryOperatorNode.Operator getRandom() { } public enum HSQLDBDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), - LIKE("LIKE"), NOT_LIKE("NOT LIKE"), SIMILAR_TO("SIMILAR TO"), NOT_SIMILAR_TO("NOT SIMILAR TO"), - REGEX_POSIX("~"), REGEX_POSIT_NOT("!~"); + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="); private String textRepr; @@ -180,4 +181,18 @@ public String getTextRepresentation() { } } + + @Override + public List> generateOrderBys() { + List> expressions = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + ArrayList hsqldbColumns = new ArrayList<>(columns); + for (int i = 0; i < nr && !hsqldbColumns.isEmpty(); i++) { + HSQLDBSchema.HSQLDBColumn randomColumn = Randomly.fromList(hsqldbColumns); + HSQLDBColumnReference columnReference = new HSQLDBColumnReference(randomColumn); + hsqldbColumns.remove(randomColumn); + expressions.add(columnReference); + } + return expressions; + } } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index 7963aca90..b4b419b45 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -2,6 +2,7 @@ import java.util.List; +import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -37,12 +38,13 @@ private SQLQueryAdapter getQuery() { sb.append(columns.get(i).getName()); sb.append("="); Node expr; - // if (Randomly.getBooleanWithSmallProbability()) { - // expr = gen.generateExpression(columns.get(i).getType()); - // HSQLDBErrors.addExpressionErrors(errors); - // } else { - expr = gen.generateConstant(columns.get(i).getType()); - // } + if (Randomly.getBooleanWithSmallProbability()) { + sb.append(" WHERE "); + expr = gen.generateExpression(columns.get(i).getType()); + // HSQLDBErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(columns.get(i).getType()); + } sb.append(HSQLDBToStringVisitor.asString(expr)); } return new SQLQueryAdapter(sb.toString(), EXPECTED_ERRORS); diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java index d6d3d553e..ec501ab5e 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; @@ -13,8 +14,9 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBToStringVisitor; import sqlancer.hsqldb.ast.HSQLDBExpression; -import sqlancer.hsqldb.ast.HSQLDBJoin; +//import sqlancer.hsqldb.ast.HSQLDBJoin; import sqlancer.hsqldb.ast.HSQLDBSelect; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; @@ -50,10 +52,13 @@ public void check() throws Exception { List> tableList = targetTables.stream() .map(t -> new TableReferenceNode(t)) .collect(Collectors.toList()); - List> joins = HSQLDBJoin.getJoins(tableList, state); - select.setJoinList(joins.stream().collect(Collectors.toList())); + // List> joins = HSQLDBJoin.getJoins(tableList, state); + // select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); + + ComparatorHelper.getResultSetFirstColumnAsString(HSQLDBToStringVisitor.asString(select), errors, state); + } List> generateFetchColumns() { diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java index 09f51051d..0b1706583 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java @@ -5,7 +5,6 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.duckdb.test.DuckDBQueryPartitioningBase; import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; import sqlancer.hsqldb.HSQLDBToStringVisitor; @@ -18,7 +17,6 @@ public HSQLDBQueryPartitioningWhereTester(HSQLDBGlobalState state) { @Override public void check() throws Exception { super.check(); - select.setWhereClause(null); String originalQueryString = HSQLDBToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); @@ -37,7 +35,7 @@ public void check() throws Exception { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state); } } From a601c52aa58e452e2ec31e62d31c9cf26a0438ce Mon Sep 17 00:00:00 2001 From: everpcpc Date: Wed, 30 Nov 2022 09:08:38 +0800 Subject: [PATCH 245/856] fix: test databend with github actions service image --- .github/workflows/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 67ade8e58..6ae3bd74e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -120,6 +120,15 @@ jobs: databend: name: DBMS Tests (Databend) runs-on: ubuntu-latest + services: + databend: + image: datafuselabs/databend + env: + DATABEND_DEFAULT_USER: sqlancer + DATABEND_DEFAULT_PASSWORD: sqlancer + ports: + - 8000:8000 + - 3307:3307 steps: - uses: actions/checkout@v2 with: @@ -130,14 +139,6 @@ jobs: java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up Databend - run: | - sudo apt update - sudo apt install mysql-client - docker pull datafuselabs/databend-query - docker run --net=host datafuselabs/databend-query & - - name: Create SQLancer user - run: mysql -uroot -h127.0.0.1 -P3307 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" - name: Run Tests run: | DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test From 0c2ed8c09c7fad29c6e125a49770e74a47e8f954 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 05:57:11 +0000 Subject: [PATCH 246/856] build(deps): bump checkstyle from 10.3.4 to 10.5.0 Bumps [checkstyle](https://github.com/checkstyle/checkstyle) from 10.3.4 to 10.5.0. - [Release notes](https://github.com/checkstyle/checkstyle/releases) - [Commits](https://github.com/checkstyle/checkstyle/compare/checkstyle-10.3.4...checkstyle-10.5.0) --- updated-dependencies: - dependency-name: com.puppycrawl.tools:checkstyle dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index ffee5dfd9..950ea4243 100644 --- a/pom.xml +++ b/pom.xml @@ -209,7 +209,7 @@ com.puppycrawl.tools checkstyle - 10.3.4 + 10.5.0 From f09e98f4dad567a61ec2dca97062bb9d2a13a3ff Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Thu, 1 Dec 2022 20:37:07 +0800 Subject: [PATCH 247/856] chore(databend):code tidying and local optimization --- src/sqlancer/ProviderAdapter.java | 8 +--- src/sqlancer/databend/DatabendErrors.java | 11 +++-- src/sqlancer/databend/DatabendProvider.java | 8 ++-- src/sqlancer/databend/DatabendSchema.java | 10 +---- .../gen/DatabendRandomQuerySynthesizer.java | 43 +++++++++++-------- .../databend/gen/DatabendTableGenerator.java | 30 ++----------- .../databend/gen/DatabendViewGenerator.java | 9 ---- .../databend/test/DatabendNoRECOracle.java | 24 ++++------- test/sqlancer/dbms/TestDatabend.java | 4 +- 9 files changed, 53 insertions(+), 94 deletions(-) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index a16f8388c..46475f260 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -45,12 +45,8 @@ public void generateAndTestDatabase(G globalState) throws Exception { for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { assert localState != null; - try { - oracle.check(); - globalState.getManager().incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - - } + oracle.check(); + globalState.getManager().incrementSelectQueryCount(); assert localState != null; localState.executedWithoutError(); } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 7e1f80c4e..ab9724a5f 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -9,24 +9,29 @@ private DatabendErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Division by zero"); - + errors.add("/ by zero"); + errors.add("ORDER BY position"); + errors.add("GROUP BY position"); /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE * databend2; CREATE TABLE t0(c0VARCHAR VARCHAR NULL, c1VARCHAR VARCHAR NULL, c2FLOAT FLOAT NOT NULL - * DEFAULT(NULL)); CREATE TABLE t1(c0INT BIGINT NULL); INSERT INTO t0(c1varchar, c0varchar) VALUES ('067596', - * '19'), ('', '87'); + * DEFAULT(NULL)); CREATE TABLE t1(c0INT BIGINT NULL); INSERT INTO t0(c1varchar, c0varchar) VALUES + * ('067596','19'), ('', '87'); */ errors.add("Can't cast column from null into non-nullable type"); } public static void addInsertErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); } public static void addGroupByErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); + errors.add("GROUP BY position"); } } diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 9a94241d0..0ce3b7695 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -23,6 +23,7 @@ import sqlancer.databend.gen.DatabendInsertGenerator; import sqlancer.databend.gen.DatabendRandomQuerySynthesizer; import sqlancer.databend.gen.DatabendTableGenerator; +import sqlancer.databend.gen.DatabendViewGenerator; @AutoService(DatabaseProvider.class) public class DatabendProvider extends SQLProviderAdapter { @@ -38,8 +39,7 @@ public enum Action implements AbstractAction { // DELETE(DatabendDeleteGenerator::generate), // // UPDATE(DatabendUpdateGenerator::getQuery), // - // CREATE_VIEW(DatabendViewGenerator::generate), //TODO 等待databend的create view语法 更加贴近mysql - EXPLAIN((g) -> { + CREATE_VIEW(DatabendViewGenerator::generate), EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); DatabendErrors.addGroupByErrors(errors); @@ -73,8 +73,8 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); // case DELETE: // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); - // case CREATE_VIEW: //TODO 暂时关闭create view - // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + case CREATE_VIEW: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: throw new AssertionError(a); } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 98d7acafd..0e12689ee 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -10,7 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.DBMSCommon; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; @@ -237,9 +236,6 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con, databaseName); for (String tableName : tableNames) { - if (DBMSCommon.matchesIndexName(tableName)) { - continue; // TODO: unexpected? - } List databaseColumns = getTableColumns(con, tableName, databaseName); boolean isView = tableName.startsWith("v"); DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); @@ -292,11 +288,7 @@ private static List getTableColumns(SQLConnection con, String ta } } } - // if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { - // TODO: implement an option to enable/disable rowids - // columns.add(new DatabendColumn("rowid", new DatabendCompositeDataType(DatabendDataType.INT, 4), false, - // false)); - // } + return columns; } diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 286764c6e..70422b324 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -5,10 +5,12 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.ast.DatabendConstant; @@ -23,38 +25,42 @@ private DatabendRandomQuerySynthesizer() { public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); - DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState) - .setColumns(targetTables.getColumns()); + List targetColumns = targetTables.getColumns(); + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(targetColumns); DatabendSelect select = new DatabendSelect(); // TODO distinct - // select.setDistinct(Randomly.getBoolean()); + select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); List> columns = new ArrayList<>(); + int freeColumns = targetColumns.size(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); + Node expression = null; + if (freeColumns > 0 && Randomly.getBoolean()) { + expression = new ColumnReferenceNode<>(targetColumns.get(freeColumns - 1)); + freeColumns -= 1; + } else { + expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); + } columns.add(expression); - // } else { - // columns.add(gen()); - // } } select.setFetchColumns(columns); List tables = targetTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); List> joins = DatabendJoin.getJoins(tableList, globalState); - select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setJoinList(joins); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - // if (Randomly.getBoolean()) {//TODO order by超过实际行数 - // select.setOrderByExpressions(gen.generateOrderBys()); - // } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } - // if (Randomly.getBoolean()) { //TODO group by超过实际行数 - // select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - // } + if (Randomly.getBoolean()) { // TODO 该表达式生成的类型较多，需要对其限制 + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } if (Randomly.getBoolean()) { select.setLimitClause( @@ -64,10 +70,11 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setOffsetClause( DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } - // TODO 待添加HavingClause - // if (Randomly.getBoolean()) { - // select.setHavingClause(gen.generateHavingClause()); - // } + + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + return select; } diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index c83cd93e9..32505f98e 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -34,27 +34,14 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(columns.get(i).getName()); sb.append(" "); sb.append(columns.get(i).getType()); - // if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() - // && columns.get(i).getType().getPrimitiveDataType() == DatabendDataType.VARCHAR) { - // sb.append(" COLLATE "); - // sb.append(getRandomCollate()); - // } - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" UNIQUE"); - // } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints && Randomly.getBooleanWithRatherLowProbability()) { sb.append(" NOT NULL"); } else { sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 } - // if (globalState.getDbmsSpecificOptions().testCheckConstraints //databend 无check约束 - // && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" CHECK("); - // sb.append(DatabendToStringVisitor.asString(gen.generateExpression())); - // DatabendErrors.addExpressionErrors(errors); - // sb.append(")"); - // } + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 @@ -62,22 +49,11 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(")"); } } - // databend并没有索引 - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { - // errors.add("Invalid type for index"); - // List primaryKeyColumns = Randomly.nonEmptySubset(columns); - // sb.append(", PRIMARY KEY("); - // sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - // sb.append(")"); - // } + sb.append(")"); return new SQLQueryAdapter(sb.toString(), errors, true); } - public static String getRandomCollate() { - return Randomly.fromOptions("NOCASE", "NOACCENT", "NOACCENT.NOCASE", "C", "POSIX"); - } - private static List getNewColumns() { List columns = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { diff --git a/src/sqlancer/databend/gen/DatabendViewGenerator.java b/src/sqlancer/databend/gen/DatabendViewGenerator.java index 55cc6ffc5..75ad57239 100644 --- a/src/sqlancer/databend/gen/DatabendViewGenerator.java +++ b/src/sqlancer/databend/gen/DatabendViewGenerator.java @@ -17,15 +17,6 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { StringBuilder sb = new StringBuilder("CREATE "); sb.append("VIEW "); sb.append(globalState.getSchema().getFreeViewName()); - // sb.append("("); - // for (int i = 0; i < nrColumns; i++) { - // if (i != 0) { - // sb.append(", "); - // } - // sb.append("c"); - // sb.append(i); - // } - // sb.append(") AS "); sb.append(" AS "); sb.append(DatabendToStringVisitor .asString(DatabendRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index cb27fa78b..c6d9c9229 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -3,7 +3,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -48,15 +47,14 @@ public void check() throws SQLException { DatabendTables randomTables = s.getRandomTableNonEmptyTables(); // 随机获得nr张表 List columns = randomTables.getColumns(); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - - Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件，形式为ast - + Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件 List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); List> joins = DatabendJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); // 禁用优化 - int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, + int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), + randomWhereCondition, joins); + int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { throw new IgnoreMeException(); @@ -67,23 +65,20 @@ public void check() throws SQLException { } } - private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + private int getUnoptimizedQueryCount(List> tableList, + Node randomWhereCondition, List> joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); - // DatabendExpression isTrue = DatabendPostfixOperation.create(randomWhereCondition, - // PostfixOperator.IS_TRUE); Node asText = new NewPostfixTextNode<>(new DatabendCastOperation( new NewPostfixTextNode(randomWhereCondition, " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); - select.setFetchColumns(Arrays.asList(asText)); // ? + select.setFetchColumns(List.of(asText)); select.setFromList(tableList); select.setJoinList(joins); int secondCount = 0; unoptimizedQueryString = "SELECT SUM(count) FROM (" + DatabendToStringVisitor.asString(select) + ") as res"; - errors.add("canceling statement due to statement timeout"); SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { @@ -101,17 +96,14 @@ private int getSecondQuery(List> tableList, Node> tableList, + private int getOptimizedQueryCount(SQLConnection con, List> tableList, List columns, Node randomWhereCondition, List> joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); - // DatabendAggregate aggr = new DatabendAggregate( List> allColumns = columns.stream() .map((c) -> new ColumnReferenceNode(c)) .collect(Collectors.toList()); - // DatabendAggregateFunction.COUNT); - // select.setFetchColumns(Arrays.asList(aggr)); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index 326a43283..0c1755a54 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -17,7 +17,7 @@ public void testDatabendNoREC() { assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); } @@ -29,7 +29,7 @@ public void testDatabendTLPQueryPartitioning() { assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); } From 4d9abd417db61a14f03f584b9e0d82255624faac Mon Sep 17 00:00:00 2001 From: Nishita Dutta Date: Fri, 2 Dec 2022 17:00:15 +0530 Subject: [PATCH 248/856] Implement NoREC Oracle and fixed UPDATE statements for HSQLDB (#2) * First commit for HSQLDB * Added Update statements * Fixes Checkstyle violations and adds support for expression * Adds Binary Comparsion and Binary arithmetic expressions * Fixes failing test * Fixes where and select expressions as per hsqldb * Fixes Checkstyle violations and adds support for expression * Formatting fixes * Implement HSQLDB Select Expressions * Fix UPDATE statements and implement NoREC --- src/sqlancer/hsqldb/HSQLDBOptions.java | 10 +- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 12 +- .../hsqldb/test/HSQLDBNoRECOracle.java | 124 ++++++++++++++++++ 3 files changed, 141 insertions(+), 5 deletions(-) create mode 100644 src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index d025d4576..9cf6dc228 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -9,13 +9,15 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; +import sqlancer.hsqldb.test.HSQLDBNoRECOracle; import sqlancer.hsqldb.test.HSQLDBQueryPartitioningWhereTester; @Parameters(commandDescription = "hsqldb") public class HSQLDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") - public List oracle = List.of(HSQLDBOracleFactory.WHERE); + public List oracle = List.of(HSQLDBOracleFactory.WHERE, + HSQLDBOracleFactory.NOREC); public enum HSQLDBOracleFactory implements OracleFactory { WHERE { @@ -23,6 +25,12 @@ public enum HSQLDBOracleFactory implements OracleFactory columns = table.getRandomNonEmptyColumnSubset(); for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -38,14 +40,16 @@ private SQLQueryAdapter getQuery() { sb.append(columns.get(i).getName()); sb.append("="); Node expr; + expr = gen.generateConstant(columns.get(i).getType()); + sb.append(HSQLDBToStringVisitor.asString(expr)); if (Randomly.getBooleanWithSmallProbability()) { sb.append(" WHERE "); expr = gen.generateExpression(columns.get(i).getType()); - // HSQLDBErrors.addExpressionErrors(errors); - } else { - expr = gen.generateConstant(columns.get(i).getType()); + sb.append(HSQLDBToStringVisitor.asString(expr)); + errors.add("data type of expression is not boolean"); + HSQLDBErrors.addExpressionErrors(errors); } - sb.append(HSQLDBToStringVisitor.asString(expr)); + } return new SQLQueryAdapter(sb.toString(), EXPECTED_ERRORS); } diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java new file mode 100644 index 000000000..7de1918b4 --- /dev/null +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -0,0 +1,124 @@ +package sqlancer.hsqldb.test; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.hsqldb.HSQLDBErrors; +import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBCompositeDataType; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBDataType; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBColumnReference; +import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; + +public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { + + private final HSQLDBSchema s; + + public HSQLDBNoRECOracle(HSQLDBGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + HSQLDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + List tables = s.getDatabaseTablesRandomSubsetNotEmpty(); + List columns = tables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList()); + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state).setColumns(columns); + + Node randomWhereCondition = gen + .generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)); + + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = HSQLDBJoin.getJoins(tableList, state); + int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); // 禁用优化 + int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getSecondQuery(List> tableList, Node randomWhereCondition, + List> joins) throws SQLException { + HSQLDBSelect select = new HSQLDBSelect(); + HSQLDBColumn c = new HSQLDBColumn("COUNT(*)", null, null); + select.setFetchColumns(List.of(new HSQLDBColumnReference(c))); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + select.setJoinList(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + HSQLDBToStringVisitor.asString(select) + ") as res"; + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, List> joins) + throws SQLException { + HSQLDBSelect select = new HSQLDBSelect(); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new HSQLDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + select.setJoinList(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = HSQLDBToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} From 7560fb632dd07473a21225ef63d0138fe5dbac47 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 4 Dec 2022 16:07:58 +0800 Subject: [PATCH 249/856] feature(databend): add delete generator Signed-off-by: Yisong Han --- src/sqlancer/databend/DatabendProvider.java | 23 +++++++-------- .../databend/gen/DatabendDeleteGenerator.java | 29 +++++++++++++++++++ .../gen/DatabendNewExpressionGenerator.java | 2 +- ...abendQueryPartitioningAggregateTester.java | 21 +++++++------- 4 files changed, 50 insertions(+), 25 deletions(-) create mode 100644 src/sqlancer/databend/gen/DatabendDeleteGenerator.java diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 0ce3b7695..f22a7cec9 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -20,6 +20,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.gen.DatabendDeleteGenerator; import sqlancer.databend.gen.DatabendInsertGenerator; import sqlancer.databend.gen.DatabendRandomQuerySynthesizer; import sqlancer.databend.gen.DatabendTableGenerator; @@ -34,12 +35,12 @@ public DatabendProvider() { public enum Action implements AbstractAction { - INSERT(DatabendInsertGenerator::getQuery), // - // TODO 等待databend实现update && delete - // DELETE(DatabendDeleteGenerator::generate), // + INSERT(DatabendInsertGenerator::getQuery), + DELETE(DatabendDeleteGenerator::generate), + // TODO 等待databend实现update // UPDATE(DatabendUpdateGenerator::getQuery), // - - CREATE_VIEW(DatabendViewGenerator::generate), EXPLAIN((g) -> { + CREATE_VIEW(DatabendViewGenerator::generate), + EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); DatabendErrors.addGroupByErrors(errors); @@ -71,8 +72,8 @@ private static int mapActions(DatabendGlobalState globalState, Action a) { // TODO 等待databend实现update && delete // case UPDATE: // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); - // case DELETE: - // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); case CREATE_VIEW: return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); default: @@ -91,7 +92,7 @@ protected DatabendSchema readSchema() throws SQLException { @Override public void generateDatabase(DatabendGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + for (int i = 0; i < Randomly.fromOptions(3, 4); i++) { boolean success; do { SQLQueryAdapter qt = new DatabendTableGenerator().getQuery(globalState); @@ -107,7 +108,7 @@ public void generateDatabase(DatabendGlobalState globalState) throws Exception { throw new IgnoreMeException(); } }); - se.executeStatements(); // 在已有的表格中插入数据，原先是增删改一些数据，除了insert和explan我都去掉了 + se.executeStatements(); // 增删改一些数据（按权重随机选取算法） } @Override @@ -129,12 +130,8 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - } - try (Statement s = con.createStatement()) { s.execute("CREATE DATABASE " + databaseName); globalState.getState().logStatement("CREATE DATABASE " + databaseName); - } - try (Statement s = con.createStatement()) { s.execute("USE " + databaseName); globalState.getState().logStatement("USE " + databaseName); } diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java new file mode 100644 index 000000000..dcef8d7a8 --- /dev/null +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -0,0 +1,29 @@ +package sqlancer.databend.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendToStringVisitor; + +public class DatabendDeleteGenerator { + + public DatabendDeleteGenerator() { + } + + public static SQLQueryAdapter generate(DatabendGlobalState globalState){ + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + if(Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) + .generateExpression(DatabendDataType.BOOLEAN))); + DatabendErrors.addExpressionErrors(errors); + } + return new SQLQueryAdapter(sb.toString(),errors); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 43d38939f..a67a5d4ff 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -251,7 +251,7 @@ protected boolean canGenerateColumnOfType(DatabendDataType type) { public enum DatabendAggregateFunction { MAX(1), MIN(1), AVG(1, DatabendDataType.INT, DatabendDataType.FLOAT), COUNT(1), SUM(1, DatabendDataType.INT, DatabendDataType.FLOAT), STDDEV_POP(1), COVAR_POP(1), COVAR_SAMP(2); - // , STRING_AGG(1), STDDEV_SAMP(1),VAR_SAMP(1), VAR_POP(1) + //, *_IF, *_DISTINCT private int nrArgs; private DatabendDataType[] dataTypes; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index 59d6695ca..56f311f2b 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -53,7 +53,7 @@ public void check() throws SQLException { List> fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { - fetchColumns.add(gen.generateAggregate()); + fetchColumns.add(gen.generateAggregate()); // TODO 更换成非聚合函数 } select.setFetchColumns(Arrays.asList(aggregate)); // if (Randomly.getBooleanWithRatherLowProbability()) { @@ -181,16 +181,15 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, Node whereClause, List> joinList) { - DatabendSelect leftSelect = new DatabendSelect(); - leftSelect.setFetchColumns(aggregates); - leftSelect.setFromList(from); - leftSelect.setWhereClause(whereClause); - leftSelect.setJoinList(joinList); - // if (Randomly.getBooleanWithSmallProbability()) { - // leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); //TODO group by超过实际行数 - // leftSelect.setGroupByExpressions(select.getFetchColumns());// TODO group by不能放入聚合函数 - // } - return leftSelect; + DatabendSelect select = new DatabendSelect(); + select.setFetchColumns(aggregates); + select.setFromList(from); + select.setWhereClause(whereClause); + select.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + select.setGroupByExpressions(List.of(gen.generateConstant(DatabendDataType.INT))); // TODO 仍可加强 + } + return select; } } From 625204e01fb4a02e8de3e715cdee2a37515b9018 Mon Sep 17 00:00:00 2001 From: Kareem Shehata Date: Sun, 4 Dec 2022 16:36:52 +0800 Subject: [PATCH 250/856] Add Reducer capabilities by introducing a Reproducer that can be retrieved from TestOracle (#624) * Made TestOracle generic taking a type parameter for global state. Updated all usages of TestOracle to add this type info. * Add Reproducer class and enable getting the reproducer from a TestOracle. * Added use of Reproducer in an example Reducer and SQLite3's NoREC Oracle. * ProviderAdapter returns a Reproducer if it's available, and Main uses it to Reduce queries. Currently experimental and only outputs to stdout. * Fixing formatting * Fixing linter problems * Fixing HSLDB errors from rebase * Fixing HSQLDBNoRECOracle3 Co-authored-by: Manuel Rigger --- src/sqlancer/DatabaseProvider.java | 4 +- src/sqlancer/Main.java | 19 +++- src/sqlancer/MainOptions.java | 6 ++ src/sqlancer/OracleFactory.java | 2 +- src/sqlancer/ProviderAdapter.java | 15 +++- src/sqlancer/Reducer.java | 7 ++ src/sqlancer/Reproducer.java | 5 ++ src/sqlancer/StateToReproduce.java | 6 +- src/sqlancer/StatementReducer.java | 87 +++++++++++++++++++ src/sqlancer/arangodb/ArangoDBOptions.java | 6 +- .../test/ArangoDBQueryPartitioningBase.java | 2 +- src/sqlancer/citus/CitusOptions.java | 12 +-- src/sqlancer/citus/CitusProvider.java | 19 ++-- .../clickhouse/ClickHouseOptions.java | 12 +-- .../oracle/norec/ClickHouseNoRECOracle.java | 3 +- .../oracle/tlp/ClickHouseTLPBase.java | 2 +- .../cockroachdb/CockroachDBOptions.java | 20 ++--- .../oracle/CockroachDBNoRECOracle.java | 3 +- .../tlp/CockroachDBTLPAggregateOracle.java | 2 +- .../oracle/tlp/CockroachDBTLPBase.java | 5 +- .../common/oracle/CompositeTestOracle.java | 16 ++-- .../oracle/DocumentRemovalOracleBase.java | 2 +- src/sqlancer/common/oracle/NoRECBase.java | 2 +- .../oracle/PivotedQuerySynthesisBase.java | 2 +- .../TernaryLogicPartitioningOracleBase.java | 2 +- src/sqlancer/common/oracle/TestOracle.java | 8 +- src/sqlancer/databend/DatabendOptions.java | 18 ++-- .../databend/test/DatabendNoRECOracle.java | 2 +- ...abendQueryPartitioningAggregateTester.java | 3 +- .../test/DatabendQueryPartitioningBase.java | 5 +- ...DatabendQueryPartitioningHavingTester.java | 3 +- src/sqlancer/duckdb/DuckDBOptions.java | 18 ++-- .../duckdb/test/DuckDBNoRECOracle.java | 2 +- ...uckDBQueryPartitioningAggregateTester.java | 3 +- .../test/DuckDBQueryPartitioningBase.java | 3 +- .../DuckDBQueryPartitioningHavingTester.java | 3 +- src/sqlancer/h2/H2Options.java | 2 +- src/sqlancer/h2/H2QueryPartitioningBase.java | 2 +- src/sqlancer/hsqldb/HSQLDBOptions.java | 6 +- .../hsqldb/test/HSQLDBNoRECOracle.java | 2 +- .../test/HSQLDBQueryPartitioningBase.java | 2 +- src/sqlancer/mariadb/MariaDBOptions.java | 2 +- .../mariadb/oracle/MariaDBNoRECOracle.java | 2 +- src/sqlancer/mongodb/MongoDBOptions.java | 12 +-- .../test/MongoDBDocumentRemovalBase.java | 5 +- .../test/MongoDBQueryPartitioningBase.java | 3 +- src/sqlancer/mysql/MySQLOptions.java | 4 +- .../oracle/MySQLQueryPartitioningBase.java | 4 +- src/sqlancer/oceanbase/OceanBaseOptions.java | 6 +- .../oracle/OceanBaseNoRECOracle.java | 2 +- .../oceanbase/oracle/OceanBaseTLPBase.java | 3 +- src/sqlancer/postgres/PostgresOptions.java | 12 +-- .../postgres/oracle/PostgresNoRECOracle.java | 2 +- .../tlp/PostgresTLPAggregateOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPBase.java | 2 +- src/sqlancer/questdb/QuestDBOptions.java | 2 +- .../test/QuestDBQueryPartitioningBase.java | 3 +- src/sqlancer/sqlite3/SQLite3Options.java | 22 ++--- .../sqlite3/oracle/SQLite3Fuzzer.java | 2 +- .../sqlite3/oracle/SQLite3NoRECOracle.java | 65 +++++++++++--- .../oracle/tlp/SQLite3TLPAggregateOracle.java | 2 +- .../sqlite3/oracle/tlp/SQLite3TLPBase.java | 2 +- .../oracle/tlp/SQLite3TLPHavingOracle.java | 2 +- src/sqlancer/tidb/TiDBOptions.java | 10 +-- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 2 +- .../tidb/oracle/TiDBTLPHavingOracle.java | 2 +- src/sqlancer/yugabyte/ycql/YCQLOptions.java | 2 +- .../yugabyte/ycql/test/YCQLFuzzer.java | 2 +- src/sqlancer/yugabyte/ysql/YSQLOptions.java | 16 ++-- .../yugabyte/ysql/oracle/YSQLCatalog.java | 2 +- .../yugabyte/ysql/oracle/YSQLFuzzer.java | 2 +- .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 2 +- .../oracle/tlp/YSQLTLPAggregateOracle.java | 2 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 2 +- 74 files changed, 374 insertions(+), 174 deletions(-) create mode 100644 src/sqlancer/Reducer.java create mode 100644 src/sqlancer/Reproducer.java create mode 100644 src/sqlancer/StatementReducer.java diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 72acb30d8..48afb00f5 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -24,11 +24,13 @@ public interface DatabaseProvider, O extends DBMS * @param globalState * the state created and is valid for this method call. * + * @return Reproducer if a bug is found and a reproducer is available. + * * @throws Exception * if creating the database fails. * */ - void generateAndTestDatabase(G globalState) throws Exception; + Reproducer generateAndTestDatabase(G globalState) throws Exception; C createDatabase(G globalState) throws Exception; diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 7953386b7..dfb2bb89b 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -314,13 +314,30 @@ public void run() throws Exception { if (options.logEachSelect()) { logger.writeCurrent(state.getState()); } - provider.generateAndTestDatabase(state); + Reproducer reproducer = provider.generateAndTestDatabase(state); try { logger.getCurrentFileWriter().close(); logger.currentFileWriter = null; } catch (IOException e) { throw new AssertionError(e); } + if (reproducer != null && options.useReducer()) { + System.out.println("EXPERIMENTAL: Trying to reduce queries using a simple reducer."); + System.out.println("Reduced query will be output to stdout but not logs."); + G newGlobalState = createGlobalState(); + newGlobalState.setState(stateToRepro); + newGlobalState.setRandomly(r); + newGlobalState.setDatabaseName(databaseName); + newGlobalState.setMainOptions(options); + newGlobalState.setDbmsSpecificOptions(command); + QueryManager newManager = new QueryManager<>(newGlobalState); + newGlobalState.setStateLogger(new StateLogger(databaseName, provider, options)); + newGlobalState.setManager(newManager); + + Reducer reducer = new StatementReducer<>(provider); + reducer.reduce(state, reproducer, newGlobalState); + throw new AssertionError("Found a potential bug"); + } } } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 2916dbdd2..329010b39 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -107,6 +107,9 @@ public class MainOptions { @Parameter(names = "--database-prefix", description = "The prefix used for each database created") private String databasePrefix = "database"; // NOPMD + @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") + private boolean useReducer = false; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -244,4 +247,7 @@ public boolean performConnectionTest() { return useConnectionTest; } + public boolean useReducer() { + return useReducer; + } } diff --git a/src/sqlancer/OracleFactory.java b/src/sqlancer/OracleFactory.java index 897293c6b..9d6e1704b 100644 --- a/src/sqlancer/OracleFactory.java +++ b/src/sqlancer/OracleFactory.java @@ -4,7 +4,7 @@ public interface OracleFactory> { - TestOracle create(G globalState) throws Exception; + TestOracle create(G globalState) throws Exception; /** * Indicates whether the test oracle requires that all tables (including views) contain at least one row. diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index a16f8388c..1a530e93e 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -35,13 +35,13 @@ public Class getOptionClass() { } @Override - public void generateAndTestDatabase(G globalState) throws Exception { + public Reproducer generateAndTestDatabase(G globalState) throws Exception { try { generateDatabase(globalState); checkViewsAreValid(globalState); globalState.getManager().incrementCreateDatabase(); - TestOracle oracle = getTestOracle(globalState); + TestOracle oracle = getTestOracle(globalState); for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { assert localState != null; @@ -50,6 +50,12 @@ public void generateAndTestDatabase(G globalState) throws Exception { globalState.getManager().incrementSelectQueryCount(); } catch (IgnoreMeException e) { + } catch (AssertionError e) { + Reproducer reproducer = oracle.getLastReproducer(); + if (reproducer != null) { + return reproducer; + } + throw e; } assert localState != null; localState.executedWithoutError(); @@ -58,11 +64,12 @@ public void generateAndTestDatabase(G globalState) throws Exception { } finally { globalState.getConnection().close(); } + return null; } protected abstract void checkViewsAreValid(G globalState); - protected TestOracle getTestOracle(G globalState) throws Exception { + protected TestOracle getTestOracle(G globalState) throws Exception { List> testOracleFactory = globalState.getDbmsSpecificOptions() .getTestOracleFactory(); boolean testOracleRequiresMoreThanZeroRows = testOracleFactory.stream() @@ -75,7 +82,7 @@ protected TestOracle getTestOracle(G globalState) throws Exception { if (testOracleFactory.size() == 1) { return testOracleFactory.get(0).create(globalState); } else { - return new CompositeTestOracle(testOracleFactory.stream().map(o -> { + return new CompositeTestOracle(testOracleFactory.stream().map(o -> { try { return o.create(globalState); } catch (Exception e1) { diff --git a/src/sqlancer/Reducer.java b/src/sqlancer/Reducer.java new file mode 100644 index 000000000..0e6589262 --- /dev/null +++ b/src/sqlancer/Reducer.java @@ -0,0 +1,7 @@ +package sqlancer; + +public interface Reducer> { + + void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception; + +} diff --git a/src/sqlancer/Reproducer.java b/src/sqlancer/Reproducer.java new file mode 100644 index 000000000..ef64bd0fe --- /dev/null +++ b/src/sqlancer/Reproducer.java @@ -0,0 +1,5 @@ +package sqlancer; + +public interface Reproducer> { + boolean bugStillTriggers(G globalState); +} diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index f6408ce58..7ac3e9ccf 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -9,7 +9,7 @@ public class StateToReproduce { - private final List> statements = new ArrayList<>(); + private List> statements = new ArrayList<>(); private final String databaseName; @@ -128,4 +128,8 @@ public OracleRunReproductionState createLocalState() { return new OracleRunReproductionState(); } + public void setStatements(List> statements) { + this.statements = statements; + } + } diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java new file mode 100644 index 000000000..14db4ba5b --- /dev/null +++ b/src/sqlancer/StatementReducer.java @@ -0,0 +1,87 @@ +package sqlancer; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiFunction; +import java.util.stream.Collectors; + +import sqlancer.common.query.Query; + +public class StatementReducer, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> + implements Reducer { + private final DatabaseProvider provider; + private boolean observedChange; + + public StatementReducer(DatabaseProvider provider) { + this.provider = provider; + } + + @SuppressWarnings("unchecked") + @Override + public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + + List> knownToReproduceBugStatements = new ArrayList<>(); + for (Query stat : state.getState().getStatements()) { + knownToReproduceBugStatements.add((Query) stat); + } + System.out.println("Starting query:"); + printQueries(knownToReproduceBugStatements); + System.out.println(); + + do { + observedChange = false; + knownToReproduceBugStatements = tryReduction(state, reproducer, newGlobalState, + knownToReproduceBugStatements, (candidateStatements, i) -> { + candidateStatements.remove((int) i); + return true; + }); + } while (observedChange); + + System.out.println("Reduced query:"); + printQueries(knownToReproduceBugStatements); + } + + private List> tryReduction(G state, // NOPMD + Reproducer reproducer, G newGlobalState, List> knownToReproduceBugStatements, + BiFunction>, Integer, Boolean> reductionOperation) throws Exception { + + List> statements = knownToReproduceBugStatements; + for (int i = 0; i < statements.size(); i++) { + try (C con2 = provider.createDatabase(newGlobalState)) { + newGlobalState.setConnection(con2); + List> candidateStatements = new ArrayList<>(statements); + if (!reductionOperation.apply(candidateStatements, i)) { + continue; + } + newGlobalState.getState().setStatements(candidateStatements.stream().collect(Collectors.toList())); + for (Query s : candidateStatements) { + try { + s.execute(newGlobalState); + } catch (Throwable ignoredException) { + // ignore + } + } + try { + if (reproducer.bugStillTriggers(newGlobalState)) { + observedChange = true; + statements = candidateStatements; + // reproducer.outputHook((SQLite3GlobalState) newGlobalState); + // state.getLogger().logReduced(newGlobalState.getState()); + } + } catch (Throwable ignoredException) { + + } + } + } + return statements; + } + + private void printQueries(List> statements) { + System.out.println("==============================="); + for (Query q : statements) { + System.out.println(q.getLogString()); + } + System.out.println("==============================="); + } + +} diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java index b821fe8c3..7e20883d5 100644 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ b/src/sqlancer/arangodb/ArangoDBOptions.java @@ -40,10 +40,10 @@ public List getTestOracleFactory() { public enum ArangoDBOracleFactory implements OracleFactory { QUERY_PARTITIONING { @Override - public TestOracle create(ArangoDBGlobalState globalState) throws Exception { - List oracles = new ArrayList<>(); + public TestOracle create(ArangoDBGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); oracles.add(new ArangoDBQueryPartitioningWhereTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } } } diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java index f583ed04f..859ffc74e 100644 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java +++ b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java @@ -17,7 +17,7 @@ public class ArangoDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, ArangoDBProvider.ArangoDBGlobalState> - implements TestOracle { + implements TestOracle { protected ArangoDBSchema schema; protected List targetColumns; diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index cdeadb672..67b14c304 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -29,21 +29,21 @@ public class CitusOptions extends PostgresOptions { public enum CitusOracleFactory implements OracleFactory { NOREC { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; return new CitusNoRECOracle(citusGlobalState); } }, PQS { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { return new PostgresPivotedQuerySynthesisOracle(globalState); } }, HAVING { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; return new CitusTLPHavingOracle(citusGlobalState); } @@ -51,13 +51,13 @@ public TestOracle create(PostgresGlobalState globalState) throws SQLException { }, QUERY_PARTITIONING { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; - List oracles = new ArrayList<>(); + List> oracles = new ArrayList<>(); oracles.add(new CitusTLPWhereOracle(citusGlobalState)); oracles.add(new CitusTLPHavingOracle(citusGlobalState)); oracles.add(new CitusTLPAggregateOracle(citusGlobalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index aaebd6709..c897d503e 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -319,15 +319,16 @@ public void generateDatabase(PostgresGlobalState globalState) throws Exception { } @Override - protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { - List oracles = ((CitusOptions) globalState.getDbmsSpecificOptions()).citusOracle.stream().map(o -> { - try { - return o.create(globalState); - } catch (Exception e1) { - throw new AssertionError(e1); - } - }).collect(Collectors.toList()); - return new CompositeTestOracle(oracles, globalState); + protected TestOracle getTestOracle(PostgresGlobalState globalState) throws SQLException { + List> oracles = ((CitusOptions) globalState + .getDbmsSpecificOptions()).citusOracle.stream().map(o -> { + try { + return o.create(globalState); + } catch (Exception e1) { + throw new AssertionError(e1); + } + }).collect(Collectors.toList()); + return new CompositeTestOracle(oracles, globalState); } private List readCitusWorkerNodes(PostgresGlobalState globalState, SQLConnection con) diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 8a2056c1b..0aafab84d 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -34,37 +34,37 @@ public class ClickHouseOptions implements DBMSSpecificOptions { TLPWhere { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPWhereOracle(globalState); } }, TLPDistinct { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPDistinctOracle(globalState); } }, TLPGroupBy { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPGroupByOracle(globalState); } }, TLPAggregate { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPAggregateOracle(globalState); } }, TLPHaving { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseTLPHavingOracle(globalState); } }, NoREC { @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { return new ClickHouseNoRECOracle(globalState); } }; diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java index c7d56ab99..cda529cbe 100644 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -28,7 +28,8 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -public class ClickHouseNoRECOracle extends NoRECBase implements TestOracle { +public class ClickHouseNoRECOracle extends NoRECBase + implements TestOracle { private final ClickHouseSchema s; diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index a9b09fd92..690994ec0 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -26,7 +26,7 @@ import sqlancer.common.oracle.TestOracle; public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { ClickHouseSchema schema; ClickHouseTables targetTables; diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index aa41259b0..38b845781 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -34,59 +34,59 @@ public class CockroachDBOptions implements DBMSSpecificOptions { NOREC { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBNoRECOracle(globalState); } }, AGGREGATE { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPAggregateOracle(globalState); } }, GROUP_BY { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPGroupByOracle(globalState); } }, HAVING { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPHavingOracle(globalState); } }, WHERE { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPWhereOracle(globalState); } }, DISTINCT { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPDistinctOracle(globalState); } }, EXTENDED_WHERE { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { return new CockroachDBTLPExtendedWhereOracle(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new CockroachDBTLPAggregateOracle(globalState)); oracles.add(new CockroachDBTLPHavingOracle(globalState)); oracles.add(new CockroachDBTLPWhereOracle(globalState)); oracles.add(new CockroachDBTLPGroupByOracle(globalState)); oracles.add(new CockroachDBTLPExtendedWhereOracle(globalState)); oracles.add(new CockroachDBTLPDistinctOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 3020f3238..77ecbcbed 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -29,7 +29,8 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -public class CockroachDBNoRECOracle extends NoRECBase implements TestOracle { +public class CockroachDBNoRECOracle extends NoRECBase + implements TestOracle { private CockroachDBExpressionGenerator gen; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index bc2601ef5..0368f5222 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -35,7 +35,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -public class CockroachDBTLPAggregateOracle implements TestOracle { +public class CockroachDBTLPAggregateOracle implements TestOracle { private final CockroachDBGlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index 7e7f03e26..21914eae2 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -22,8 +22,9 @@ import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; -public class CockroachDBTLPBase extends - TernaryLogicPartitioningOracleBase implements TestOracle { +public class CockroachDBTLPBase + extends TernaryLogicPartitioningOracleBase + implements TestOracle { CockroachDBSchema s; CockroachDBTables targetTables; diff --git a/src/sqlancer/common/oracle/CompositeTestOracle.java b/src/sqlancer/common/oracle/CompositeTestOracle.java index fbb66960a..0fdc13140 100644 --- a/src/sqlancer/common/oracle/CompositeTestOracle.java +++ b/src/sqlancer/common/oracle/CompositeTestOracle.java @@ -4,27 +4,27 @@ import sqlancer.GlobalState; -public class CompositeTestOracle implements TestOracle { +public class CompositeTestOracle> implements TestOracle { - private final TestOracle[] oracles; - private final GlobalState globalState; + private final List> oracles; + private final G globalState; private int i; - public CompositeTestOracle(List oracles, GlobalState globalState) { + public CompositeTestOracle(List> oracles, G globalState) { this.globalState = globalState; - this.oracles = oracles.toArray(new TestOracle[oracles.size()]); + this.oracles = oracles; } @Override public void check() throws Exception { try { - oracles[i].check(); - boolean lastOracleIndex = i == oracles.length - 1; + oracles.get(i).check(); + boolean lastOracleIndex = i == oracles.size() - 1; if (!lastOracleIndex) { globalState.getManager().incrementSelectQueryCount(); } } finally { - i = (i + 1) % oracles.length; + i = (i + 1) % oracles.size(); } } } diff --git a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java b/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java index a48a6f17e..b6c0ee509 100644 --- a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java +++ b/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java @@ -3,7 +3,7 @@ import sqlancer.GlobalState; import sqlancer.common.gen.ExpressionGenerator; -public abstract class DocumentRemovalOracleBase> implements TestOracle { +public abstract class DocumentRemovalOracleBase> implements TestOracle { protected E predicate; diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java index 734e04a86..c099b2592 100644 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ b/src/sqlancer/common/oracle/NoRECBase.java @@ -6,7 +6,7 @@ import sqlancer.SQLGlobalState; import sqlancer.common.query.ExpectedErrors; -public abstract class NoRECBase> implements TestOracle { +public abstract class NoRECBase> implements TestOracle { protected final S state; protected final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index cd2580b76..6c551d18e 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -12,7 +12,7 @@ import sqlancer.common.schema.AbstractRowValue; public abstract class PivotedQuerySynthesisBase, R extends AbstractRowValue, E, C extends SQLancerDBConnection> - implements TestOracle { + implements TestOracle { protected final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java index 3b5d87814..b4d6add4b 100644 --- a/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java +++ b/src/sqlancer/common/oracle/TernaryLogicPartitioningOracleBase.java @@ -14,7 +14,7 @@ * @param * the global state type */ -public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { +public abstract class TernaryLogicPartitioningOracleBase> implements TestOracle { protected E predicate; protected E negatedPredicate; diff --git a/src/sqlancer/common/oracle/TestOracle.java b/src/sqlancer/common/oracle/TestOracle.java index 737adb692..2dfd854a0 100644 --- a/src/sqlancer/common/oracle/TestOracle.java +++ b/src/sqlancer/common/oracle/TestOracle.java @@ -1,7 +1,13 @@ package sqlancer.common.oracle; -public interface TestOracle { +import sqlancer.GlobalState; +import sqlancer.Reproducer; + +public interface TestOracle> { void check() throws Exception; + default Reproducer getLastReproducer() { + return null; + } } diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index ebacd21cf..efcdec4eb 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -99,53 +99,53 @@ public enum DatabendOracleFactory implements OracleFactory NOREC { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendNoRECOracle(globalState); } }, HAVING { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningHavingTester(globalState); } }, WHERE { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningWhereTester(globalState); } }, GROUP_BY { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningGroupByTester(globalState); } }, AGGREGATE { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningAggregateTester(globalState); } }, DISTINCT { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { + public TestOracle create(DatabendGlobalState globalState) throws SQLException { return new DatabendQueryPartitioningDistinctTester(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(DatabendGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new DatabendQueryPartitioningWhereTester(globalState)); oracles.add(new DatabendQueryPartitioningHavingTester(globalState)); oracles.add(new DatabendQueryPartitioningAggregateTester(globalState)); oracles.add(new DatabendQueryPartitioningDistinctTester(globalState)); oracles.add(new DatabendQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index cb27fa78b..79c369f12 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -33,7 +33,7 @@ import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendNoRECOracle extends NoRECBase implements TestOracle { +public class DatabendNoRECOracle extends NoRECBase implements TestOracle { private final DatabendSchema s; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java index 59d6695ca..b6e423980 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java @@ -30,7 +30,8 @@ import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; import sqlancer.databend.gen.DatabendNewExpressionGenerator.DatabendAggregateFunction; -public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { +public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase + implements TestOracle { private String firstResult; private String secondResult; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java index 6ef132572..2242bf99f 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java @@ -24,8 +24,9 @@ import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendQueryPartitioningBase extends - TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { +public class DatabendQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, DatabendGlobalState> + implements TestOracle { DatabendSchema s; DatabendTables targetTables; diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java index 8ca1489f2..a64be36b0 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java @@ -16,7 +16,8 @@ import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; -public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase implements TestOracle { +public class DatabendQueryPartitioningHavingTester extends DatabendQueryPartitioningBase + implements TestOracle { public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { super(state); diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index 4e1c49cd1..f14379163 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -97,53 +97,53 @@ public enum DuckDBOracleFactory implements OracleFactory { NOREC { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBNoRECOracle(globalState); } }, HAVING { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningHavingTester(globalState); } }, WHERE { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningWhereTester(globalState); } }, GROUP_BY { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningGroupByTester(globalState); } }, AGGREGATE { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningAggregateTester(globalState); } }, DISTINCT { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { return new DuckDBQueryPartitioningDistinctTester(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new DuckDBQueryPartitioningWhereTester(globalState)); oracles.add(new DuckDBQueryPartitioningHavingTester(globalState)); oracles.add(new DuckDBQueryPartitioningAggregateTester(globalState)); oracles.add(new DuckDBQueryPartitioningDistinctTester(globalState)); oracles.add(new DuckDBQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 550043615..de89bfa7d 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -33,7 +33,7 @@ import sqlancer.duckdb.gen.DuckDBExpressionGenerator; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBCastOperation; -public class DuckDBNoRECOracle extends NoRECBase implements TestOracle { +public class DuckDBNoRECOracle extends NoRECBase implements TestOracle { private final DuckDBSchema s; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index a6b8d8f27..8d7835d29 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -30,7 +30,8 @@ import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPostfixOperator; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBUnaryPrefixOperator; -public class DuckDBQueryPartitioningAggregateTester extends DuckDBQueryPartitioningBase implements TestOracle { +public class DuckDBQueryPartitioningAggregateTester extends DuckDBQueryPartitioningBase + implements TestOracle { private String firstResult; private String secondResult; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 68456ef26..837247f62 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -24,7 +24,8 @@ import sqlancer.duckdb.gen.DuckDBExpressionGenerator; public class DuckDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, DuckDBGlobalState> implements TestOracle { + extends TernaryLogicPartitioningOracleBase, DuckDBGlobalState> + implements TestOracle { DuckDBSchema s; DuckDBTables targetTables; diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index fc8843729..aef003fa5 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -14,7 +14,8 @@ import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBQueryPartitioningHavingTester extends DuckDBQueryPartitioningBase implements TestOracle { +public class DuckDBQueryPartitioningHavingTester extends DuckDBQueryPartitioningBase + implements TestOracle { public DuckDBQueryPartitioningHavingTester(DuckDBGlobalState state) { super(state); diff --git a/src/sqlancer/h2/H2Options.java b/src/sqlancer/h2/H2Options.java index 45b47f203..e7141fcf2 100644 --- a/src/sqlancer/h2/H2Options.java +++ b/src/sqlancer/h2/H2Options.java @@ -20,7 +20,7 @@ public enum H2OracleFactory implements OracleFactory { TLP_WHERE { @Override - public TestOracle create(H2GlobalState globalState) throws SQLException { + public TestOracle create(H2GlobalState globalState) throws SQLException { return new H2QueryPartitioningWhereTester(globalState); } diff --git a/src/sqlancer/h2/H2QueryPartitioningBase.java b/src/sqlancer/h2/H2QueryPartitioningBase.java index b15f1c312..ec5af4e2e 100644 --- a/src/sqlancer/h2/H2QueryPartitioningBase.java +++ b/src/sqlancer/h2/H2QueryPartitioningBase.java @@ -17,7 +17,7 @@ import sqlancer.h2.H2Schema.H2Tables; public class H2QueryPartitioningBase extends TernaryLogicPartitioningOracleBase, H2GlobalState> - implements TestOracle { + implements TestOracle { H2Schema s; H2Tables targetTables; diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index 9cf6dc228..3e3dcc4dc 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -22,13 +22,15 @@ public class HSQLDBOptions implements DBMSSpecificOptions { WHERE { @Override - public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) throws SQLException { + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws SQLException { return new HSQLDBQueryPartitioningWhereTester(globalState); } }, NOREC { @Override - public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) throws Exception { + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws Exception { return new HSQLDBNoRECOracle(globalState); } } diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java index 7de1918b4..ca376e348 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -31,7 +31,7 @@ import sqlancer.hsqldb.ast.HSQLDBSelect; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { +public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { private final HSQLDBSchema s; diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java index ec501ab5e..5e62a39ac 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -22,7 +22,7 @@ public class HSQLDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> - implements TestOracle { + implements TestOracle { HSQLDBSelect select; HSQLDBExpressionGenerator expressionGenerator; diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 02e6eab64..70e0aedbd 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -28,7 +28,7 @@ public enum MariaDBOracleFactory implements OracleFactory { NOREC { @Override - public TestOracle create(MariaDBGlobalState globalState) throws SQLException { + public TestOracle create(MariaDBGlobalState globalState) throws SQLException { return new MariaDBNoRECOracle(globalState); } diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index 882b49c08..bba0627e4 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -27,7 +27,7 @@ import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; -public class MariaDBNoRECOracle extends NoRECBase implements TestOracle { +public class MariaDBNoRECOracle extends NoRECBase implements TestOracle { private final MariaDBSchema s; private static final int NOT_FOUND = -1; diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java index 9e02de67b..78bd73ffc 100644 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ b/src/sqlancer/mongodb/MongoDBOptions.java @@ -56,18 +56,18 @@ public List getTestOracleFactory() { public enum MongoDBOracleFactory implements OracleFactory { QUERY_PARTITIONING { @Override - public TestOracle create(MongoDBGlobalState globalState) throws Exception { - List oracles = new ArrayList<>(); + public TestOracle create(MongoDBGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); oracles.add(new MongoDBQueryPartitioningWhereTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }, DOCUMENT_REMOVAL { @Override - public TestOracle create(MongoDBGlobalState globalState) throws Exception { - List oracles = new ArrayList<>(); + public TestOracle create(MongoDBGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); oracles.add(new MongoDBDocumentRemovalTester(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } } } diff --git a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java index 6faae42bc..bf39643c1 100644 --- a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java +++ b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java @@ -15,8 +15,9 @@ import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator; import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; -public class MongoDBDocumentRemovalBase extends - DocumentRemovalOracleBase, MongoDBProvider.MongoDBGlobalState> implements TestOracle { +public class MongoDBDocumentRemovalBase + extends DocumentRemovalOracleBase, MongoDBProvider.MongoDBGlobalState> + implements TestOracle { protected MongoDBSchema schema; protected MongoDBSchema.MongoDBTables targetTables; diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java index 30ebe56bd..ea6d39bbf 100644 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java @@ -19,7 +19,8 @@ import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; public class MongoDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, MongoDBGlobalState> implements TestOracle { + extends TernaryLogicPartitioningOracleBase, MongoDBGlobalState> + implements TestOracle { protected MongoDBSchema schema; protected MongoDBTables targetTables; diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 2a3f16968..ad8060090 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -28,7 +28,7 @@ public enum MySQLOracleFactory implements OracleFactory { TLP_WHERE { @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { + public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLTLPWhereOracle(globalState); } @@ -36,7 +36,7 @@ public TestOracle create(MySQLGlobalState globalState) throws SQLException { PQS { @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { + public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLPivotedQuerySynthesisOracle(globalState); } diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java index 8946136fe..7b7ba8b34 100644 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java @@ -19,8 +19,8 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.gen.MySQLExpressionGenerator; -public abstract class MySQLQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase implements TestOracle { +public abstract class MySQLQueryPartitioningBase extends + TernaryLogicPartitioningOracleBase implements TestOracle { MySQLSchema s; MySQLTables targetTables; diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index b19de3983..86507f769 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -28,20 +28,20 @@ public enum OceanBaseOracleFactory implements OracleFactory create(OceanBaseGlobalState globalState) throws SQLException { return new OceanBaseTLPWhereOracle(globalState); } }, NoREC { @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { return new OceanBaseNoRECOracle(globalState); } }, PQS { @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { return new OceanBasePivotedQuerySynthesisOracle(globalState); } diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java index 900ac8caa..cc98dcf0a 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java @@ -28,7 +28,7 @@ import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { +public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { // SELECT COUNT(*) FROM t0 WHERE ; // SELECT SUM(count) FROM (SELECT IS TRUE as count FROM t0); diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java index 51f06a3fc..73ff19bfb 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java @@ -21,7 +21,8 @@ import sqlancer.oceanbase.gen.OceanBaseHintGenerator; public abstract class OceanBaseTLPBase - extends TernaryLogicPartitioningOracleBase implements TestOracle { + extends TernaryLogicPartitioningOracleBase + implements TestOracle { OceanBaseSchema s; OceanBaseTables targetTables; diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index b6c79d752..948a539ce 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -44,13 +44,13 @@ public class PostgresOptions implements DBMSSpecificOptions { NOREC { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { return new PostgresNoRECOracle(globalState); } }, PQS { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { return new PostgresPivotedQuerySynthesisOracle(globalState); } @@ -62,19 +62,19 @@ public boolean requiresAllTablesToContainRows() { HAVING { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { + public TestOracle create(PostgresGlobalState globalState) throws SQLException { return new PostgresTLPHavingOracle(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new PostgresTLPWhereOracle(globalState)); oracles.add(new PostgresTLPHavingOracle(globalState)); oracles.add(new PostgresTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 995eb207f..f4bc10e74 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -36,7 +36,7 @@ import sqlancer.postgres.gen.PostgresExpressionGenerator; import sqlancer.postgres.oracle.tlp.PostgresTLPBase; -public class PostgresNoRECOracle extends NoRECBase implements TestOracle { +public class PostgresNoRECOracle extends NoRECBase implements TestOracle { private final PostgresSchema s; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index b9ea695e2..a1eaa43c7 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -29,7 +29,7 @@ import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.gen.PostgresCommon; -public class PostgresTLPAggregateOracle extends PostgresTLPBase implements TestOracle { +public class PostgresTLPAggregateOracle extends PostgresTLPBase implements TestOracle { private String firstResult; private String secondResult; diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 34f5e602a..5e349b5dc 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -29,7 +29,7 @@ import sqlancer.postgres.oracle.PostgresNoRECOracle; public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { protected PostgresSchema s; protected PostgresTables targetTables; diff --git a/src/sqlancer/questdb/QuestDBOptions.java b/src/sqlancer/questdb/QuestDBOptions.java index 527e80cc8..f84ca3d47 100644 --- a/src/sqlancer/questdb/QuestDBOptions.java +++ b/src/sqlancer/questdb/QuestDBOptions.java @@ -27,7 +27,7 @@ public enum QuestDBOracleFactory implements OracleFactory { // TODO (anxing): implement test oracles WHERE { @Override - public TestOracle create(QuestDBGlobalState globalState) throws SQLException { + public TestOracle create(QuestDBGlobalState globalState) throws SQLException { return new QuestDBQueryPartitioningWhereTester(globalState); } } diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index fb6bd3c3f..45e3db9c4 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -24,7 +24,8 @@ import sqlancer.questdb.gen.QuestDBExpressionGenerator; public class QuestDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, QuestDBGlobalState> implements TestOracle { + extends TernaryLogicPartitioningOracleBase, QuestDBGlobalState> + implements TestOracle { QuestDBSchema s; QuestDBTables targetTables; diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index dabc3ce13..6df9c378c 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -93,7 +93,7 @@ public class SQLite3Options implements DBMSSpecificOptions public enum SQLite3OracleFactory implements OracleFactory { PQS { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3PivotedQuerySynthesisOracle(globalState); } @@ -105,14 +105,14 @@ public boolean requiresAllTablesToContainRows() { }, NoREC { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3NoRECOracle(globalState); } }, AGGREGATE { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPAggregateOracle(globalState); } @@ -120,45 +120,45 @@ public TestOracle create(SQLite3GlobalState globalState) throws SQLException { WHERE { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPWhereOracle(globalState); } }, DISTINCT { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPDistinctOracle(globalState); } }, GROUP_BY { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPGroupByOracle(globalState); } }, HAVING { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3TLPHavingOracle(globalState); } }, FUZZER { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { return new SQLite3Fuzzer(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new SQLite3TLPWhereOracle(globalState)); oracles.add(new SQLite3TLPDistinctOracle(globalState)); oracles.add(new SQLite3TLPGroupByOracle(globalState)); oracles.add(new SQLite3TLPHavingOracle(globalState)); oracles.add(new SQLite3TLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index 8e868dfd0..bf13ffc80 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -7,7 +7,7 @@ import sqlancer.sqlite3.SQLite3Visitor; // tries to trigger a crash -public class SQLite3Fuzzer implements TestOracle { +public class SQLite3Fuzzer implements TestOracle { private final SQLite3GlobalState globalState; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index db9f55531..826a2727b 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -4,9 +4,11 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.function.Function; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -29,11 +31,28 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3NoRECOracle extends NoRECBase implements TestOracle { +public class SQLite3NoRECOracle extends NoRECBase implements TestOracle { private static final int NO_VALID_RESULT = -1; private final SQLite3Schema s; private SQLite3ExpressionGenerator gen; + private Reproducer reproducer; + + private class SQLite3NoRECReproducer implements Reproducer { + private final Function optimizedQuery; + private final Function unoptimizedQuery; + + SQLite3NoRECReproducer(Function optimizedQuery, + Function unoptimizedQuery) { + this.optimizedQuery = optimizedQuery; + this.unoptimizedQuery = unoptimizedQuery; + } + + @Override + public boolean bugStillTriggers(SQLite3GlobalState globalState) { + return optimizedQuery.apply(globalState) != unoptimizedQuery.apply(globalState); + } + } public SQLite3NoRECOracle(SQLite3GlobalState globalState) { super(globalState); @@ -51,6 +70,7 @@ public SQLite3NoRECOracle(SQLite3GlobalState globalState) { @Override public void check() throws SQLException { + reproducer = null; SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); gen = new SQLite3ExpressionGenerator(state).setColumns(columns); @@ -62,19 +82,28 @@ public void check() throws SQLException { select.setFromTables(tableRefs); select.setJoinClauses(joinStatements); - int optimizedCount = getOptimizedQuery(select, randomWhereCondition); - int unoptimizedCount = getUnoptimizedQuery(select, randomWhereCondition); + Function optimizedQuery = getOptimizedQuery(select, randomWhereCondition); + Function unoptimizedQuery = getUnoptimizedQuery(select, randomWhereCondition); + int optimizedCount = optimizedQuery.apply(state); + int unoptimizedCount = unoptimizedQuery.apply(state); if (optimizedCount == NO_VALID_RESULT || unoptimizedCount == NO_VALID_RESULT) { throw new IgnoreMeException(); } if (optimizedCount != unoptimizedCount) { + reproducer = new SQLite3NoRECReproducer(optimizedQuery, unoptimizedQuery); state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); throw new AssertionError(optimizedCount + " " + unoptimizedCount); } } - private int getUnoptimizedQuery(SQLite3Select select, SQLite3Expression randomWhereCondition) throws SQLException { + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + + private Function getUnoptimizedQuery(SQLite3Select select, + SQLite3Expression randomWhereCondition) throws SQLException { SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, randomWhereCondition); SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); @@ -85,10 +114,17 @@ private int getUnoptimizedQuery(SQLite3Select select, SQLite3Expression randomWh logger.writeCurrent(unoptimizedQueryString); } SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - return extractCounts(q); + return new Function() { + + @Override + public Integer apply(SQLite3GlobalState state) { + return extractCounts(q, state); + } + }; } - private int getOptimizedQuery(SQLite3Select select, SQLite3Expression randomWhereCondition) throws SQLException { + private Function getOptimizedQuery(SQLite3Select select, + SQLite3Expression randomWhereCondition) throws SQLException { boolean useAggregate = Randomly.getBoolean(); if (Randomly.getBoolean()) { select.setOrderByExpressions(gen.generateOrderBys()); @@ -106,12 +142,19 @@ private int getOptimizedQuery(SQLite3Select select, SQLite3Expression randomWher logger.writeCurrent(optimizedQueryString); } SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - return useAggregate ? extractCounts(q) : countRows(q); + return new Function() { + + @Override + public Integer apply(SQLite3GlobalState state) { + return useAggregate ? extractCounts(q, state) : countRows(q, state); + } + + }; } - private int countRows(SQLQueryAdapter q) { + private int countRows(SQLQueryAdapter q, SQLite3GlobalState globalState) { int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(state)) { + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs == null) { return NO_VALID_RESULT; } else { @@ -132,9 +175,9 @@ private int countRows(SQLQueryAdapter q) { return count; } - private int extractCounts(SQLQueryAdapter q) { + private int extractCounts(SQLQueryAdapter q, SQLite3GlobalState globalState) { int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(state)) { + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs == null) { return NO_VALID_RESULT; } else { diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index c0243425f..715770257 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -28,7 +28,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3TLPAggregateOracle implements TestOracle { +public class SQLite3TLPAggregateOracle implements TestOracle { private final SQLite3GlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index de1f01c92..650df8537 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -23,7 +23,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; public class SQLite3TLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { SQLite3Schema s; SQLite3Tables targetTables; diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index 2524d979b..6a960021d 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -29,7 +29,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3TLPHavingOracle implements TestOracle { +public class SQLite3TLPHavingOracle implements TestOracle { private final SQLite3GlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index f9137f6f7..51afa2a9a 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -29,23 +29,23 @@ public class TiDBOptions implements DBMSSpecificOptions { public enum TiDBOracleFactory implements OracleFactory { HAVING { @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { + public TestOracle create(TiDBGlobalState globalState) throws SQLException { return new TiDBTLPHavingOracle(globalState); } }, WHERE { @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { + public TestOracle create(TiDBGlobalState globalState) throws SQLException { return new TiDBTLPWhereOracle(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(TiDBGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new TiDBTLPWhereOracle(globalState)); oracles.add(new TiDBTLPHavingOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } }; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index ba2ec9df0..9e86ffd12 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -23,7 +23,7 @@ import sqlancer.tidb.gen.TiDBHintGenerator; public abstract class TiDBTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { TiDBSchema s; TiDBTables targetTables; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java index a9f64671b..68b4a56c1 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java @@ -12,7 +12,7 @@ import sqlancer.tidb.ast.TiDBExpression; import sqlancer.tidb.visitor.TiDBVisitor; -public class TiDBTLPHavingOracle extends TiDBTLPBase implements TestOracle { +public class TiDBTLPHavingOracle extends TiDBTLPBase implements TestOracle { public TiDBTLPHavingOracle(TiDBGlobalState state) { super(state); diff --git a/src/sqlancer/yugabyte/ycql/YCQLOptions.java b/src/sqlancer/yugabyte/ycql/YCQLOptions.java index 7e35e2cea..df4854757 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLOptions.java +++ b/src/sqlancer/yugabyte/ycql/YCQLOptions.java @@ -37,7 +37,7 @@ public class YCQLOptions implements DBMSSpecificOptions { public enum YCQLOracleFactory implements OracleFactory { FUZZER { @Override - public TestOracle create(YCQLGlobalState globalState) throws SQLException { + public TestOracle create(YCQLGlobalState globalState) throws SQLException { return new YCQLFuzzer(globalState); } diff --git a/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java index 8dee32c3e..7c21b7db8 100644 --- a/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java +++ b/src/sqlancer/yugabyte/ycql/test/YCQLFuzzer.java @@ -11,7 +11,7 @@ import sqlancer.yugabyte.ycql.YCQLToStringVisitor; import sqlancer.yugabyte.ycql.gen.YCQLRandomQuerySynthesizer; -public class YCQLFuzzer implements TestOracle { +public class YCQLFuzzer implements TestOracle { private final YCQLProvider.YCQLGlobalState globalState; private final List testQueries; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/yugabyte/ysql/YSQLOptions.java b/src/sqlancer/yugabyte/ysql/YSQLOptions.java index 4cd299660..6b46b09bf 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLOptions.java +++ b/src/sqlancer/yugabyte/ysql/YSQLOptions.java @@ -48,25 +48,25 @@ public List getTestOracleFactory() { public enum YSQLOracleFactory implements OracleFactory { FUZZER { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLFuzzer(globalState); } }, CATALOG { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLCatalog(globalState); } }, NOREC { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLNoRECOracle(globalState); } }, PQS { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLPivotedQuerySynthesisOracle(globalState); } @@ -77,19 +77,19 @@ public boolean requiresAllTablesToContainRows() { }, HAVING { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { + public TestOracle create(YSQLGlobalState globalState) throws SQLException { return new YSQLTLPHavingOracle(globalState); } }, QUERY_PARTITIONING { @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - List oracles = new ArrayList<>(); + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); oracles.add(new YSQLTLPWhereOracle(globalState)); oracles.add(new YSQLTLPHavingOracle(globalState)); oracles.add(new YSQLTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); + return new CompositeTestOracle(oracles, globalState); } } diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java index d6763e799..e5ae3fe1b 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLCatalog.java @@ -18,7 +18,7 @@ import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.gen.YSQLTableGenerator; -public class YSQLCatalog implements TestOracle { +public class YSQLCatalog implements TestOracle { protected final YSQLGlobalState state; protected final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java index 0aa16993e..8d0cea997 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLFuzzer.java @@ -13,7 +13,7 @@ import sqlancer.yugabyte.ysql.YSQLVisitor; import sqlancer.yugabyte.ysql.gen.YSQLRandomQueryGenerator; -public class YSQLFuzzer implements TestOracle { +public class YSQLFuzzer implements TestOracle { private final YSQLGlobalState globalState; private final List testQueries; private final ExpectedErrors errors = new ExpectedErrors(); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java index 6b2a57560..3ae27fd73 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -33,7 +33,7 @@ import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPBase; -public class YSQLNoRECOracle extends NoRECBase implements TestOracle { +public class YSQLNoRECOracle extends NoRECBase implements TestOracle { private final YSQLSchema s; diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java index 26e4d9179..537a074ae 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -29,7 +29,7 @@ import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation.PrefixOperator; import sqlancer.yugabyte.ysql.ast.YSQLSelect; -public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { +public class YSQLTLPAggregateOracle extends YSQLTLPBase implements TestOracle { private String firstResult; private String secondResult; diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java index e574391de..863e85968 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -26,7 +26,7 @@ import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; public class YSQLTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { + implements TestOracle { protected YSQLSchema s; protected YSQLTables targetTables; From f060d92551755f6c24dde4a84f31cdd560b222e4 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 5 Dec 2022 10:52:35 +0800 Subject: [PATCH 251/856] tidb: skip invalid regexp pattern (#628) Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 0586846fc..0022908a9 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -32,6 +32,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("error parsing regexp"); errors.add("from regexp"); errors.add("Empty pattern is invalid"); + errors.add("Invalid regexp pattern"); // To avoid bugs errors.add("Unknown column"); // https://github.com/pingcap/tidb/issues/35522 From f5228187b7a2d25738a846816d9c3333c1ff4764 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 11 Dec 2022 18:29:31 +0800 Subject: [PATCH 252/856] fix ci when to install citus Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6ae3bd74e..5a15ac75b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -43,9 +43,9 @@ jobs: run: | echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list curl https://install.citusdata.com/community/deb.sh | sudo bash - sudo apt-get -y install postgresql-13-citus-10.1 + sudo apt-get -y install postgresql-15-citus-11.1 sudo chown -R $USER:$USER /var/run/postgresql - export PATH=/usr/lib/postgresql/13/bin:$PATH + export PATH=/usr/lib/postgresql/15/bin:$PATH cd ~ mkdir -p citus/coordinator citus/worker1 citus/worker2 initdb -D citus/coordinator From ea05d14bf7cc59cdd6badb472208a3756b13170c Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Sun, 11 Dec 2022 22:55:12 +0800 Subject: [PATCH 253/856] feature(databend): enhance norec, tlp, and implement pqs Signed-off-by: Yisong Han --- src/sqlancer/ProviderAdapter.java | 8 +- src/sqlancer/databend/DatabendErrors.java | 4 + .../DatabendExpectedValueVisitor.java | 155 ++++++++ src/sqlancer/databend/DatabendExprToNode.java | 25 ++ src/sqlancer/databend/DatabendOptions.java | 19 +- src/sqlancer/databend/DatabendProvider.java | 6 +- src/sqlancer/databend/DatabendSchema.java | 70 +++- .../ast/DatabendAggregateOperation.java | 49 +++ .../ast/DatabendBetweenOperation.java | 40 +++ .../DatabendBinaryArithmeticOperation.java | 82 ++++- .../DatabendBinaryComparisonOperation.java | 122 ++++++- .../ast/DatabendBinaryLogicalOperation.java | 88 ++++- .../databend/ast/DatabendCastOperation.java | 28 +- .../databend/ast/DatabendColumnValue.java | 29 ++ .../databend/ast/DatabendConstant.java | 340 ++++++++++++++++-- .../databend/ast/DatabendExpression.java | 9 + .../ast/DatabendFunctionOperation.java | 4 + .../databend/ast/DatabendInOperation.java | 53 +++ src/sqlancer/databend/ast/DatabendJoin.java | 7 +- .../databend/ast/DatabendLikeOperation.java | 43 ++- .../databend/ast/DatabendOrderByTerm.java | 10 + .../ast/DatabendUnaryPostfixOperation.java | 63 ++-- .../ast/DatabendUnaryPrefixOperation.java | 88 +++-- .../databend/gen/DatabendDeleteGenerator.java | 13 +- .../databend/gen/DatabendInsertGenerator.java | 6 +- .../gen/DatabendNewExpressionGenerator.java | 179 +++++---- .../gen/DatabendRandomQuerySynthesizer.java | 51 ++- .../databend/gen/DatabendTableGenerator.java | 7 +- .../databend/test/DatabendNoRECOracle.java | 35 +- .../DatabendPivotedQuerySynthesisOracle.java | 154 ++++++++ ...abendQueryPartitioningAggregateTester.java | 20 +- .../DatabendQueryPartitioningBase.java | 27 +- ...tabendQueryPartitioningDistinctTester.java | 12 +- ...atabendQueryPartitioningGroupByTester.java | 11 +- ...DatabendQueryPartitioningHavingTester.java | 20 +- .../DatabendQueryPartitioningWhereTester.java | 9 +- test/sqlancer/dbms/TestDatabend.java | 32 +- 37 files changed, 1626 insertions(+), 292 deletions(-) create mode 100644 src/sqlancer/databend/DatabendExpectedValueVisitor.java create mode 100644 src/sqlancer/databend/DatabendExprToNode.java create mode 100644 src/sqlancer/databend/ast/DatabendAggregateOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendBetweenOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendColumnValue.java create mode 100644 src/sqlancer/databend/ast/DatabendFunctionOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendInOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendOrderByTerm.java create mode 100644 src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningAggregateTester.java (91%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningBase.java (73%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningDistinctTester.java (78%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningGroupByTester.java (85%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningHavingTester.java (80%) rename src/sqlancer/databend/test/{ => tlp}/DatabendQueryPartitioningWhereTester.java (85%) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 46475f260..a16f8388c 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -45,8 +45,12 @@ public void generateAndTestDatabase(G globalState) throws Exception { for (int i = 0; i < globalState.getOptions().getNrQueries(); i++) { try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { assert localState != null; - oracle.check(); - globalState.getManager().incrementSelectQueryCount(); + try { + oracle.check(); + globalState.getManager().incrementSelectQueryCount(); + } catch (IgnoreMeException e) { + + } assert localState != null; localState.executedWithoutError(); } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index ab9724a5f..0a12f6b84 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -12,6 +12,10 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("/ by zero"); errors.add("ORDER BY position"); errors.add("GROUP BY position"); + errors.add("downcast column error"); // bug + errors.add("index out of bounds"); // bug + errors.add("validity's length must be equal"); // bug + errors.add("validity must be equal to the array's length"); // bug /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE * databend2; CREATE TABLE t0(c0VARCHAR VARCHAR NULL, c1VARCHAR VARCHAR NULL, c2FLOAT FLOAT NOT NULL diff --git a/src/sqlancer/databend/DatabendExpectedValueVisitor.java b/src/sqlancer/databend/DatabendExpectedValueVisitor.java new file mode 100644 index 000000000..f6ff93819 --- /dev/null +++ b/src/sqlancer/databend/DatabendExpectedValueVisitor.java @@ -0,0 +1,155 @@ +package sqlancer.databend; + +import java.util.List; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; + +public class DatabendExpectedValueVisitor { + + protected final StringBuilder sb = new StringBuilder(); + + private void print(Node expr) { + sb.append(DatabendToStringVisitor.asString(expr)); + sb.append(" -- "); + sb.append(((DatabendExpression) expr).getExpectedValue()); + sb.append("\n"); + } + + @SuppressWarnings("unchecked") + public void visit(Node expr) { + assert expr != null; + if (expr instanceof ColumnReferenceNode) { + visit((ColumnReferenceNode) expr); + } else if (expr instanceof NewUnaryPostfixOperatorNode) { + visit((NewUnaryPostfixOperatorNode) expr); + } else if (expr instanceof NewUnaryPrefixOperatorNode) { + visit((NewUnaryPrefixOperatorNode) expr); + } else if (expr instanceof NewBinaryOperatorNode) { + visit((NewBinaryOperatorNode) expr); + } else if (expr instanceof TableReferenceNode) { + visit((TableReferenceNode) expr); + } else if (expr instanceof NewFunctionNode) { + visit((NewFunctionNode) expr); + } else if (expr instanceof NewBetweenOperatorNode) { + visit((NewBetweenOperatorNode) expr); + } else if (expr instanceof NewInOperatorNode) { + visit((NewInOperatorNode) expr); + } else if (expr instanceof NewOrderingTerm) { + visit((NewOrderingTerm) expr); + } else if (expr instanceof NewAliasNode) { + visit((NewAliasNode) expr); + } else if (expr instanceof NewPostfixTextNode) { + visit((NewPostfixTextNode) expr); + } else if (expr instanceof DatabendConstant) { + visit((DatabendConstant) expr); + } else if (expr instanceof DatabendSelect) { + visit((DatabendSelect) expr); + } else if (expr instanceof DatabendJoin) { + visit((DatabendJoin) expr); + } else { + throw new AssertionError(expr); + } + } + + public void visit(ColumnReferenceNode c) { + print(c); + } + + public void visit(NewUnaryPostfixOperatorNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewUnaryPrefixOperatorNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewBinaryOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(TableReferenceNode t) { + print(t); + } + + public void visit(NewFunctionNode fun) { + print(fun); + visit(fun.getArgs()); + } + + public void visit(List> expressions) { + for (Node expression : expressions) { + visit(expression); + } + } + + public void visit(NewBetweenOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getMiddle()); + visit(op.getRight()); + } + + public void visit(NewInOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(NewOrderingTerm op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewAliasNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewPostfixTextNode postFixText) { + print(postFixText); + visit(postFixText.getExpr()); + } + + public void visit(DatabendConstant constant) { + print(constant); + } + + public void visit(DatabendSelect select) { + print(select.getWhereClause()); + } + + public void visit(DatabendJoin join) { + print(join.getOnCondition()); + } + + public String get() { + return sb.toString(); + } + + public static String asExpectedValues(Node expr) { + DatabendExpectedValueVisitor v = new DatabendExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + +} diff --git a/src/sqlancer/databend/DatabendExprToNode.java b/src/sqlancer/databend/DatabendExprToNode.java new file mode 100644 index 000000000..e81269893 --- /dev/null +++ b/src/sqlancer/databend/DatabendExprToNode.java @@ -0,0 +1,25 @@ +package sqlancer.databend; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.newast.Node; +import sqlancer.databend.ast.DatabendExpression; + +public class DatabendExprToNode { + + private DatabendExprToNode() { + + } + + @SuppressWarnings("unchecked") + public static Node cast(DatabendExpression expression) { + return (Node) expression; + } + + @SuppressWarnings("unchecked") + public static List> casts(List expressions) { + return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index ebacd21cf..f88449475 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -15,11 +15,12 @@ import sqlancer.databend.DatabendOptions.DatabendOracleFactory; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.test.DatabendNoRECOracle; -import sqlancer.databend.test.DatabendQueryPartitioningAggregateTester; -import sqlancer.databend.test.DatabendQueryPartitioningDistinctTester; -import sqlancer.databend.test.DatabendQueryPartitioningGroupByTester; -import sqlancer.databend.test.DatabendQueryPartitioningHavingTester; -import sqlancer.databend.test.DatabendQueryPartitioningWhereTester; +import sqlancer.databend.test.DatabendPivotedQuerySynthesisOracle; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningAggregateTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningDistinctTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningGroupByTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningHavingTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningWhereTester; @Parameters(commandDescription = "Databend") public class DatabendOptions implements DBMSSpecificOptions { @@ -147,7 +148,13 @@ public TestOracle create(DatabendGlobalState globalState) throws SQLException { oracles.add(new DatabendQueryPartitioningGroupByTester(globalState)); return new CompositeTestOracle(oracles, globalState); } - }; + }, + PQS { + @Override + public TestOracle create(DatabendGlobalState globalState) throws Exception { + return new DatabendPivotedQuerySynthesisOracle(globalState); + } + } } diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index f22a7cec9..0ab0d4c9b 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -35,12 +35,10 @@ public DatabendProvider() { public enum Action implements AbstractAction { - INSERT(DatabendInsertGenerator::getQuery), - DELETE(DatabendDeleteGenerator::generate), + INSERT(DatabendInsertGenerator::getQuery), DELETE(DatabendDeleteGenerator::generate), // TODO 等待databend实现update // UPDATE(DatabendUpdateGenerator::getQuery), // - CREATE_VIEW(DatabendViewGenerator::generate), - EXPLAIN((g) -> { + CREATE_VIEW(DatabendViewGenerator::generate), EXPLAIN((g) -> { ExpectedErrors errors = new ExpectedErrors(); DatabendErrors.addExpressionErrors(errors); DatabendErrors.addGroupByErrors(errors); diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 0e12689ee..c82ad82ca 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -1,22 +1,29 @@ package sqlancer.databend; +import static sqlancer.databend.DatabendSchema.DatabendDataType.INT; + import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.ast.DatabendConstant; public class DatabendSchema extends AbstractSchema { @@ -158,6 +165,55 @@ public DatabendTables(List tables) { super(tables); } + public DatabendRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String rowValueQuery = String.format("SELECT %s FROM %s ORDER BY 1 LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet rs = s.executeQuery(rowValueQuery); + if (!rs.next()) { + throw new AssertionError("could not find random row " + rowValueQuery + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + DatabendColumn column = getColumns().get(i); + int columnIndex = rs.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + DatabendConstant constant; + if (rs.getString(columnIndex) == null) { + constant = DatabendConstant.createNullConstant(); + } else { + switch (column.getType().getPrimitiveDataType()) { + case INT: + constant = DatabendConstant.createIntConstant(rs.getLong(columnIndex)); + break; + case BOOLEAN: + constant = DatabendConstant.createBooleanConstant(rs.getBoolean(columnIndex)); + break; + case VARCHAR: + constant = DatabendConstant.createStringConstant(rs.getString(columnIndex)); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !rs.next(); + return new DatabendRowValue(this, values); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } + + } + + public static class DatabendRowValue extends AbstractRowValue { + + DatabendRowValue(DatabendTables tables, Map values) { + super(tables, values); + } + } public DatabendSchema(List databaseTables) { @@ -168,6 +224,12 @@ public DatabendTables getRandomTableNonEmptyTables() { return new DatabendTables(Randomly.nonEmptySubset(getDatabaseTables())); } + public DatabendTables getRandomTableNonEmptyAndViewTables() { + List tables = getDatabaseTables().stream().filter(t -> !t.isView()).collect(Collectors.toList()); + tables = Randomly.nonEmptySubset(tables); + return new DatabendTables(tables); + } + private static DatabendCompositeDataType getColumnType(String typeString) { DatabendDataType primitiveType; int size = -1; @@ -176,19 +238,19 @@ private static DatabendCompositeDataType getColumnType(String typeString) { } switch (typeString) { case "INT": - primitiveType = DatabendDataType.INT; + primitiveType = INT; size = 4; break; case "SMALLINT": - primitiveType = DatabendDataType.INT; + primitiveType = INT; size = 2; break; case "BIGINT": - primitiveType = DatabendDataType.INT; + primitiveType = INT; size = 8; break; case "TINYINT": - primitiveType = DatabendDataType.INT; + primitiveType = INT; size = 1; break; case "VARCHAR": diff --git a/src/sqlancer/databend/ast/DatabendAggregateOperation.java b/src/sqlancer/databend/ast/DatabendAggregateOperation.java new file mode 100644 index 000000000..df5e96199 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendAggregateOperation.java @@ -0,0 +1,49 @@ +package sqlancer.databend.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendAggregateOperation + extends NewFunctionNode + implements DatabendExpression { + public DatabendAggregateOperation(List args, DatabendAggregateFunction func) { + super(DatabendExprToNode.casts(args), func); + } + + public enum DatabendAggregateFunction { + MAX(1), MIN(1), AVG(1, DatabendSchema.DatabendDataType.INT, DatabendSchema.DatabendDataType.FLOAT), COUNT(1), + SUM(1, DatabendSchema.DatabendDataType.INT, DatabendSchema.DatabendDataType.FLOAT), STDDEV_POP(1), COVAR_POP(1), + COVAR_SAMP(2); + // , *_IF, *_DISTINCT + + private int nrArgs; + private DatabendSchema.DatabendDataType[] dataTypes; + + DatabendAggregateFunction(int nrArgs, DatabendSchema.DatabendDataType... dataTypes) { + this.nrArgs = nrArgs; + this.dataTypes = dataTypes.clone(); + } + + public static DatabendAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public DatabendSchema.DatabendDataType getRandomType() { + if (dataTypes.length == 0) { + return Randomly.fromOptions(DatabendSchema.DatabendDataType.values()); + } else { + return Randomly.fromOptions(dataTypes); + } + } + + public int getNrArgs() { + return nrArgs; + } + + } + +} diff --git a/src/sqlancer/databend/ast/DatabendBetweenOperation.java b/src/sqlancer/databend/ast/DatabendBetweenOperation.java new file mode 100644 index 000000000..27f5532c4 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendBetweenOperation.java @@ -0,0 +1,40 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendBetweenOperation extends NewBetweenOperatorNode implements DatabendExpression { + public DatabendBetweenOperation(DatabendExpression left, DatabendExpression middle, DatabendExpression right, + boolean isTrue) { + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(middle), DatabendExprToNode.cast(right), isTrue); + } + + public DatabendExpression getLeftExpr() { + return (DatabendExpression) left; + } + + public DatabendExpression getMiddleExpr() { + return (DatabendExpression) middle; + } + + public DatabendExpression getRightExpr() { + return (DatabendExpression) right; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendBinaryComparisonOperation leftComparison = new DatabendBinaryComparisonOperation(getMiddleExpr(), + getLeftExpr(), DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator.LESS_EQUALS); + DatabendBinaryComparisonOperation rightComparison = new DatabendBinaryComparisonOperation(getLeftExpr(), + getRightExpr(), DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator.LESS_EQUALS); + return new DatabendBinaryLogicalOperation(leftComparison, rightComparison, + DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator.AND).getExpectedValue(); + } + + @Override + public DatabendSchema.DatabendDataType getExpectedType() { + return DatabendSchema.DatabendDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java index dfc44ee22..f052b3ccb 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -1,18 +1,51 @@ package sqlancer.databend.ast; +import java.util.function.BinaryOperator; + import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode { +public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode + implements DatabendExpression { - public DatabendBinaryArithmeticOperation(Node left, Node right, + public DatabendBinaryArithmeticOperation(DatabendExpression left, DatabendExpression right, BinaryOperatorNode.Operator op) { - super(left, right, op); + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); } public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Operator { - ADDITION("+"), SUBTRACTION("-"), MULTIPLICATION("*"), DIVISION("/"), MODULO("%"); + ADDITION("+") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> l + r); + } + }, + SUBTRACTION("-") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + } + }, + MODULO("%") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + } + }; private final String textRepresentation; @@ -20,10 +53,49 @@ public enum DatabendBinaryArithmeticOperator implements BinaryOperatorNode.Opera textRepresentation = text; } + public abstract DatabendConstant apply(DatabendConstant left, DatabendConstant right); + + public DatabendConstant applyOperation(DatabendConstant left, DatabendConstant right, BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + long leftVal = left.cast(DatabendDataType.INT).asInt(); + long rightVal = right.cast(DatabendDataType.INT).asInt(); + return DatabendConstant.createIntConstant(op.apply(leftVal, rightVal)); + } + } + @Override public String getTextRepresentation() { return textRepresentation; } } + public DatabendExpression getLeftExpr() { + return (DatabendExpression) super.getLeft(); + } + + public DatabendExpression getRightExpr() { + return (DatabendExpression) super.getRight(); + } + + public DatabendBinaryArithmeticOperator getOp() { + return (DatabendBinaryArithmeticOperator) op; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftValue = getLeftExpr().getExpectedValue(); + DatabendConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.INT; + } + } diff --git a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java index cf63ad17e..4f72dbc84 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryComparisonOperation.java @@ -2,18 +2,126 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode { +public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode + implements DatabendExpression { - public DatabendBinaryComparisonOperation(Node left, Node right, + public DatabendBinaryComparisonOperation(DatabendExpression left, DatabendExpression right, DatabendBinaryComparisonOperator op) { - super(left, right, op); + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); + } + + public DatabendExpression getLeftExpression() { + return (DatabendExpression) super.getLeft(); + } + + public DatabendExpression getRightExpression() { + return (DatabendExpression) super.getRight(); + } + + public DatabendBinaryComparisonOperator getOp() { + return (DatabendBinaryComparisonOperator) op; + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftExpectedValue = getLeftExpression().getExpectedValue(); + DatabendConstant rightExpectedValue = getRightExpression().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); } public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Operator { - EQUALS("="), IS_DISTINCT("IS DISTINCT FROM"), IS_NOT_DISTINCT("IS NOT DISTINCT FROM"), NOT_EQUALS("!="), - LESS("<"), LESS_EQUALS("<="), GREATER(">"), GREATER_EQUALS(">="); + EQUALS("=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return left.isEquals(right); + } + }, + NOT_EQUALS("!=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isEquals = left.isEquals(right); + if (isEquals.isBoolean()) { + return DatabendConstant.createBooleanConstant(!isEquals.asBoolean()); + } + return isEquals; + } + }, + IS_DISTINCT("IS DISTINCT FROM") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return DatabendConstant.createBooleanConstant(!IS_NOT_DISTINCT.apply(left, right).asBoolean()); + } + }, + IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + if (left.isNull()) { + return DatabendConstant.createBooleanConstant(right.isNull()); + } else if (right.isNull()) { + return DatabendConstant.createBooleanConstant(false); + } else { + return left.isEquals(right); + } + } + }, + LESS("<") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + return left.isLessThan(right); + } + }, + LESS_EQUALS("<=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isLessThan = left.isLessThan(right); + if (isLessThan.isBoolean() && !isLessThan.asBoolean()) { + return left.isEquals(right); + } else { + return isLessThan; + } + } + }, + GREATER(">") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isEquals = left.isEquals(right); + if (isEquals.isBoolean() && isEquals.asBoolean()) { + return DatabendConstant.createBooleanConstant(false); + } else { + DatabendConstant less = left.isLessThan(right); + if (less.isNull()) { + return DatabendConstant.createNullConstant(); + } + return DatabendConstant.createBooleanConstant(!less.asBoolean()); + } + } + }, + GREATER_EQUALS(">=") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant isEquals = left.isEquals(right); + if (isEquals.isBoolean() && isEquals.asBoolean()) { + return DatabendConstant.createBooleanConstant(true); + } else { + DatabendConstant less = left.isLessThan(right); + if (less.isNull()) { + return DatabendConstant.createNullConstant(); + } + return DatabendConstant.createBooleanConstant(!less.asBoolean()); + } + } + }; private final String textRepresentation; @@ -21,6 +129,8 @@ public enum DatabendBinaryComparisonOperator implements BinaryOperatorNode.Opera textRepresentation = text; } + public abstract DatabendConstant apply(DatabendConstant left, DatabendConstant right); + @Override public String getTextRepresentation() { return textRepresentation; diff --git a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java index 355ec88a5..4ee7aa705 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryLogicalOperation.java @@ -3,17 +3,92 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode { +public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode + implements DatabendExpression { - public DatabendBinaryLogicalOperation(Node left, Node right, + public DatabendBinaryLogicalOperation(DatabendExpression left, DatabendExpression right, DatabendBinaryLogicalOperator op) { - super(left, right, op); + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); + } + + public DatabendExpression getLeftExpr() { + return (DatabendExpression) super.getLeft(); + } + + public DatabendExpression getRightExpr() { + return (DatabendExpression) super.getRight(); + } + + public DatabendBinaryLogicalOperator getOp() { + return (DatabendBinaryLogicalOperator) op; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftValue = getLeftExpr().getExpectedValue(); + DatabendConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; } public enum DatabendBinaryLogicalOperator implements BinaryOperatorNode.Operator { - AND("AND", "and"), OR("OR", "or"); + AND("AND", "and") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant leftVal = left.cast(DatabendDataType.BOOLEAN); + DatabendConstant rightVal = right.cast(DatabendDataType.BOOLEAN); + assert (leftVal.isNull() || leftVal.isBoolean()) : leftVal + "不是NULL也不是Boolean类型"; + assert (rightVal.isNull() || rightVal.isBoolean()) : rightVal + "不是NULL也不是Boolean类型"; + if (leftVal.isNull()) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + if (rightVal.asBoolean()) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(false); + } + } + } else if (!leftVal.asBoolean()) { + return DatabendConstant.createBooleanConstant(false); + } + assert leftVal.asBoolean(); + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(rightVal.asBoolean()); + } + } + }, + OR("OR", "or") { + @Override + public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { + DatabendConstant leftVal = left.cast(DatabendDataType.BOOLEAN); + DatabendConstant rightVal = right.cast(DatabendDataType.BOOLEAN); + assert (leftVal.isNull() || leftVal.isBoolean()) : leftVal + "不是NULL也不是Boolean类型"; + assert (rightVal.isNull() || rightVal.isBoolean()) : rightVal + "不是NULL也不是Boolean类型"; + if (leftVal.isBoolean() && leftVal.asBoolean()) { + return DatabendConstant.createBooleanConstant(true); + } + if (rightVal.isBoolean() && rightVal.asBoolean()) { + return DatabendConstant.createBooleanConstant(true); + } + if (leftVal.isNull() || rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } + return DatabendConstant.createBooleanConstant(false); + } + }; private final String[] textRepresentations; @@ -33,6 +108,9 @@ public DatabendBinaryLogicalOperator getRandomOp() { public static DatabendBinaryLogicalOperator getRandom() { return Randomly.fromOptions(values()); } + + public abstract DatabendConstant apply(DatabendConstant left, DatabendConstant right); + } } diff --git a/src/sqlancer/databend/ast/DatabendCastOperation.java b/src/sqlancer/databend/ast/DatabendCastOperation.java index c1b3ffe66..534836d59 100644 --- a/src/sqlancer/databend/ast/DatabendCastOperation.java +++ b/src/sqlancer/databend/ast/DatabendCastOperation.java @@ -3,17 +3,39 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.ast.newast.Node; -import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendCastOperation extends NewUnaryPostfixOperatorNode { +public class DatabendCastOperation extends NewUnaryPostfixOperatorNode + implements DatabendExpression { - public DatabendCastOperation(Node expr, DatabendSchema.DatabendCompositeDataType type) { + DatabendDataType type; + + public DatabendCastOperation(Node expr, DatabendCompositeDataType type) { super(expr, new BinaryOperatorNode.Operator() { @Override public String getTextRepresentation() { return "::" + type.toString(); } }); + this.type = type.getPrimitiveDataType(); + } + + DatabendExpression getExpression() { + return (DatabendExpression) getExpr(); } + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type); + } + + @Override + public DatabendDataType getExpectedType() { + return type; + } } diff --git a/src/sqlancer/databend/ast/DatabendColumnValue.java b/src/sqlancer/databend/ast/DatabendColumnValue.java new file mode 100644 index 000000000..b6067a050 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendColumnValue.java @@ -0,0 +1,29 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendDataType; + +public class DatabendColumnValue extends ColumnReferenceNode + implements DatabendExpression { + + private final DatabendConstant expectedValue; + + public DatabendColumnValue(DatabendColumn column, DatabendConstant value) { + super(column); + this.expectedValue = value; + } + + public DatabendConstant getExpectedValue() { + return expectedValue; + } + + public DatabendDataType getExpectedType() { + return getColumn().getType().getPrimitiveDataType(); + } + + public static DatabendColumnValue create(DatabendColumn column, DatabendConstant value) { + return new DatabendColumnValue(column, value); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 39056aff9..0cc3d0bd2 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -4,12 +4,57 @@ import java.text.SimpleDateFormat; import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendConstant implements Node { +public abstract class DatabendConstant implements Node, DatabendExpression { private DatabendConstant() { } + public boolean isNull() { + return false; + } + + public boolean isInt() { + return false; + } + + public boolean isBoolean() { + return false; + } + + public boolean isString() { + return false; + } + + public boolean isFloat() { + return false; + } + + public abstract DatabendConstant cast(DatabendDataType dataType); + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asFloat() { + throw new UnsupportedOperationException(this.toString()); + } + + public abstract DatabendConstant isEquals(DatabendConstant rightVal); + + public abstract DatabendConstant isLessThan(DatabendConstant rightVal); + + // public abstract String getTextRepresentation(); + public static class DatabendNullConstant extends DatabendConstant { @Override @@ -17,6 +62,35 @@ public String toString() { return "NULL"; } + @Override + public boolean isNull() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + return DatabendConstant.createNullConstant(); + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return DatabendConstant.createNullConstant(); + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + return DatabendConstant.createNullConstant(); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.NULL; + } + + // @Override + // public DatabendConstant getExpectedValue() { + // return super.getExpectedValue(); + // } } public static class DatabendIntConstant extends DatabendConstant { @@ -36,13 +110,66 @@ public long getValue() { return value; } + @Override + public boolean isInt() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case BOOLEAN: + return new DatabendBooleanConstant(value != 0); + case INT: + return this; + case VARCHAR: + return new DatabendStringConstant(String.valueOf(value)); + default: + return null; + } + } + + @Override + public long asInt() { + return value; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DatabendConstant.createBooleanConstant(value == rightVal.asInt()); + } else { + throw new AssertionError(rightVal); + } + + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.INT; + } } - public static class DatabendDoubleConstant extends DatabendConstant { + public static class DatabendFloatConstant extends DatabendConstant { private final double value; - public DatabendDoubleConstant(double value) { + public DatabendFloatConstant(double value) { this.value = value; } @@ -50,6 +177,11 @@ public double getValue() { return value; } + @Override + public boolean isFloat() { + return true; + } + @Override public String toString() { if (value == Double.POSITIVE_INFINITY) { @@ -61,13 +193,51 @@ public String toString() { return String.valueOf(value); } + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case FLOAT: + return this; + case INT: + return DatabendConstant.createIntConstant((long) value); + case BOOLEAN: + return DatabendConstant.createBooleanConstant(value != 0); + case VARCHAR: + return DatabendConstant.createStringConstant(String.valueOf(value)); + default: + return null; + } + } + + @Override + public double asFloat() { + return value; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return null; + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DatabendConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } } - public static class DatabendTextConstant extends DatabendConstant { + public static class DatabendStringConstant extends DatabendConstant { private final String value; - public DatabendTextConstant(String value) { + public DatabendStringConstant(String value) { this.value = value; } @@ -80,25 +250,68 @@ public String toString() { return "'" + value.replace("'", "''") + "'"; } - } - - public static class DatabendBitConstant extends DatabendConstant { - - private final String value; + @Override + public String asString() { + return value; + } - public DatabendBitConstant(long value) { - this.value = Long.toBinaryString(value); + @Override + public boolean isString() { + return true; } - public String getValue() { - return value; + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case VARCHAR: + return this; + case INT: + try { + return new DatabendIntConstant(Long.parseLong(value)); + } catch (NumberFormatException e) { + return new DatabendIntConstant(-1); + } + case BOOLEAN: + if ("false".contentEquals(value.toLowerCase())) { + return new DatabendBooleanConstant(false); + } else if ("true".contentEquals(value.toLowerCase())) { + return new DatabendBooleanConstant(true); + } else { + throw new AssertionError(String.format("string: %s, cannot be forced to boolean", value)); + } + case FLOAT: + try { + return new DatabendFloatConstant(Double.parseDouble(value)); + } catch (NumberFormatException e) { + return new DatabendFloatConstant(-1); + } + default: + return null; + } } @Override - public String toString() { - return "B'" + value + "'"; + public DatabendConstant isEquals(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isString()) { + return DatabendConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + // TODO 可以比较 date和timestamp类型，待添加 + throw new AssertionError(rightVal); + } } + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isString()) { + return DatabendConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } } public static class DatabendDateConstant extends DatabendConstant { @@ -120,6 +333,20 @@ public String toString() { return String.format("DATE '%s'", textRepr); } + @Override + public DatabendConstant cast(DatabendDataType dataType) { + return null; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return null; + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + return null; + } } public static class DatabendTimestampConstant extends DatabendConstant { @@ -141,6 +368,20 @@ public String toString() { return String.format("TIMESTAMP '%s'", textRepr); } + @Override + public DatabendConstant cast(DatabendDataType dataType) { + return null; + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + return null; + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + return null; + } } public static class DatabendBooleanConstant extends DatabendConstant { @@ -160,33 +401,82 @@ public String toString() { return String.valueOf(value); } + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public DatabendConstant cast(DatabendDataType dataType) { + switch (dataType) { + case BOOLEAN: + return this; + case INT: + return new DatabendIntConstant(value ? 1 : 0); + case FLOAT: + return new DatabendFloatConstant(value ? 1 : 0); + case VARCHAR: + return new DatabendStringConstant(value ? "1" : "0"); + default: + return null; + } + } + + @Override + public DatabendConstant isEquals(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return DatabendConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else { + throw new AssertionError(rightVal); + } + + } + + @Override + public DatabendConstant isLessThan(DatabendConstant rightVal) { + if (rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return DatabendConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } else { + throw new AssertionError(rightVal); + } + } + } - public static Node createStringConstant(String text) { - return new DatabendTextConstant(text); + public static DatabendConstant createStringConstant(String text) { + return new DatabendStringConstant(text); } - public static Node createFloatConstant(double val) { - return new DatabendDoubleConstant(val); + public static DatabendConstant createFloatConstant(double val) { + return new DatabendFloatConstant(val); } - public static Node createIntConstant(long val) { + public static DatabendConstant createIntConstant(long val) { return new DatabendIntConstant(val); } - public static Node createNullConstant() { + public static DatabendConstant createNullConstant() { return new DatabendNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static DatabendConstant createBooleanConstant(boolean val) { return new DatabendBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static DatabendConstant createDateConstant(long integer) { return new DatabendDateConstant(integer); } - public static Node createTimestampConstant(long integer) { + public static DatabendConstant createTimestampConstant(long integer) { return new DatabendTimestampConstant(integer); } diff --git a/src/sqlancer/databend/ast/DatabendExpression.java b/src/sqlancer/databend/ast/DatabendExpression.java index 31b9bf859..d53952140 100644 --- a/src/sqlancer/databend/ast/DatabendExpression.java +++ b/src/sqlancer/databend/ast/DatabendExpression.java @@ -1,5 +1,14 @@ package sqlancer.databend.ast; +import sqlancer.databend.DatabendSchema.DatabendDataType; + public interface DatabendExpression { + default DatabendDataType getExpectedType() { + return null; + } + + default DatabendConstant getExpectedValue() { + return null; + } } diff --git a/src/sqlancer/databend/ast/DatabendFunctionOperation.java b/src/sqlancer/databend/ast/DatabendFunctionOperation.java new file mode 100644 index 000000000..855e1c8f1 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendFunctionOperation.java @@ -0,0 +1,4 @@ +package sqlancer.databend.ast; + +public class DatabendFunctionOperation { // TODO add databend function +} diff --git a/src/sqlancer/databend/ast/DatabendInOperation.java b/src/sqlancer/databend/ast/DatabendInOperation.java new file mode 100644 index 000000000..304c3e3f9 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendInOperation.java @@ -0,0 +1,53 @@ +package sqlancer.databend.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendInOperation extends NewInOperatorNode implements DatabendExpression { + + private final DatabendExpression leftExpr; + private final List rightExpr; + + public DatabendInOperation(DatabendExpression left, List right, boolean isNegated) { + super(DatabendExprToNode.cast(left), DatabendExprToNode.casts(right), isNegated); + this.leftExpr = left; + this.rightExpr = right; + } + + @Override + public DatabendSchema.DatabendDataType getExpectedType() { + return DatabendSchema.DatabendDataType.BOOLEAN; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftValue = leftExpr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return DatabendConstant.createNullConstant(); + } + boolean isNull = false; + for (DatabendExpression expr : rightExpr) { + DatabendConstant rightValue = expr.getExpectedValue(); + if (rightValue == null) { + return null; + } + if (rightValue.isNull()) { + isNull = true; + } else if (rightValue.isEquals(leftValue).isBoolean() && rightValue.isEquals(leftValue).asBoolean()) { + return DatabendConstant.createBooleanConstant(!isNegated()); + } + } + + if (isNull) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(isNegated()); + } + } +} diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index 899929873..123916542 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -6,6 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -83,18 +84,18 @@ public static List> getJoins( switch (JoinType.getRandom()) { case INNER: joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); break; case NATURAL: joinExpressions.add(DatabendJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); break; case LEFT: joinExpressions.add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); break; case RIGHT: joinExpressions.add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, - joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); break; default: throw new AssertionError(); diff --git a/src/sqlancer/databend/ast/DatabendLikeOperation.java b/src/sqlancer/databend/ast/DatabendLikeOperation.java index 2755970e7..36ac73603 100644 --- a/src/sqlancer/databend/ast/DatabendLikeOperation.java +++ b/src/sqlancer/databend/ast/DatabendLikeOperation.java @@ -1,15 +1,48 @@ package sqlancer.databend.ast; +import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendLikeOperation extends NewBinaryOperatorNode { +public class DatabendLikeOperation extends NewBinaryOperatorNode implements DatabendExpression { - public DatabendLikeOperation(Node left, Node right, - DatabendLikeOperator op) { - super(left, right, op); + public DatabendLikeOperation(DatabendExpression left, DatabendExpression right, DatabendLikeOperator op) { + super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); + } + + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; + } + + public DatabendExpression getLeftExpr() { + return (DatabendExpression) super.getLeft(); + } + + public DatabendExpression getRightExpr() { + return (DatabendExpression) super.getRight(); + } + + public DatabendLikeOperator getOp() { + return (DatabendLikeOperator) op; + } + + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant leftVal = getLeftExpr().getExpectedValue(); + DatabendConstant rightVal = getRightExpr().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } + if (leftVal.isNull() || rightVal.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + boolean result = LikeImplementationHelper.match(leftVal.asString(), rightVal.asString(), 0, 0, true); + return DatabendConstant.createBooleanConstant(result); + } } public enum DatabendLikeOperator implements BinaryOperatorNode.Operator { diff --git a/src/sqlancer/databend/ast/DatabendOrderByTerm.java b/src/sqlancer/databend/ast/DatabendOrderByTerm.java new file mode 100644 index 000000000..5c78bae3f --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendOrderByTerm.java @@ -0,0 +1,10 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.Node; + +public class DatabendOrderByTerm extends NewOrderingTerm implements DatabendExpression { + public DatabendOrderByTerm(Node expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java index e01c35282..a19512edf 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -3,23 +3,36 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { +public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements DatabendExpression { - // private final Node expr; - // private final DatabendUnaryPostfixOperator op; - private boolean negate; + public DatabendUnaryPostfixOperation(DatabendExpression expr, DatabendUnaryPostfixOperator op) { + super(DatabendExprToNode.cast(expr), op); + } + + public DatabendExpression getExpression() { + return (DatabendExpression) getExpr(); + } + + public DatabendUnaryPostfixOperator getOp() { + return (DatabendUnaryPostfixOperator) op; + } - public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op, - boolean negate) { - super(expr, op); - setNegate(negate); + @Override + public DatabendDataType getExpectedType() { + return DatabendDataType.BOOLEAN; } - public DatabendUnaryPostfixOperation(Node expr, DatabendUnaryPostfixOperator op) { - super(expr, op); + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); } public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator { @@ -28,12 +41,22 @@ public enum DatabendUnaryPostfixOperator implements BinaryOperatorNode.Operator public DatabendDataType[] getInputDataTypes() { return DatabendDataType.values(); } + + @Override + public DatabendConstant apply(DatabendConstant value) { + return DatabendConstant.createBooleanConstant(value.isNull()); + } }, IS_NOT_NULL("IS NOT NULL") { @Override public DatabendDataType[] getInputDataTypes() { return DatabendDataType.values(); } + + @Override + public DatabendConstant apply(DatabendConstant value) { + return DatabendConstant.createBooleanConstant(!value.isNull()); + } }; // IS @@ -54,19 +77,7 @@ public String getTextRepresentation() { public abstract DatabendDataType[] getInputDataTypes(); - } - - public boolean isNegated() { - return negate; - } - - public void setNegate(boolean negate) { - this.negate = negate; - } - - // @Override - public Node getExpression() { - return getExpr(); + public abstract DatabendConstant apply(DatabendConstant value); } @Override @@ -74,8 +85,4 @@ public String getOperatorRepresentation() { return this.op.getTextRepresentation(); } - // @Override - // public OperatorKind getOperatorKind() { - // return OperatorKind.POSTFIX; - // } } diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index 69c95a745..f115684e0 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -3,72 +3,88 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; -public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { +public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements DatabendExpression { - // private final Node expr; - // private final DatabendUnaryPrefixOperator op; - // private boolean negate; - - public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op) { - super(expr, op); + public DatabendUnaryPrefixOperation(DatabendExpression expr, DatabendUnaryPrefixOperator op) { + super(DatabendExprToNode.cast(expr), op); } - // public DatabendUnaryPrefixOperation(Node expr, DatabendUnaryPrefixOperator op, boolean - // negate) { - // super(expr,op); - // setNegate(negate); - // } + public DatabendExpression getExpression() { + return (DatabendExpression) getExpr(); + } - // void setNegate(boolean negate){ - // this.negate = negate; - // } + public DatabendUnaryPrefixOperator getOp() { + return (DatabendUnaryPrefixOperator) op; + } - // @Override - public Node getExpression() { - return getExpr(); + @Override + public DatabendDataType getExpectedType() { + return getOp().getExpressionType(getExpression()); } - // @Override - // public OperatorKind getOperatorKind() { - // return OperatorKind.PREFIX; - // } + @Override + public DatabendConstant getExpectedValue() { + DatabendConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } public enum DatabendUnaryPrefixOperator implements BinaryOperatorNode.Operator { NOT("NOT", DatabendDataType.BOOLEAN, DatabendDataType.INT) { @Override - public DatabendDataType getExpressionType() { + public DatabendDataType getExpressionType(DatabendExpression expr) { return DatabendDataType.BOOLEAN; } @Override - protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { - return null; // TODO + protected DatabendConstant apply(DatabendConstant value) { + if (value.isNull()) { + return DatabendConstant.createNullConstant(); + } else { + return DatabendConstant.createBooleanConstant(!value.cast(DatabendDataType.BOOLEAN).asBoolean()); + } } }, UNARY_PLUS("+", DatabendDataType.INT) { @Override - public DatabendDataType getExpressionType() { - return DatabendDataType.INT; + public DatabendDataType getExpressionType(DatabendExpression expr) { + return expr.getExpectedType(); } @Override - protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { - return expectedValue; + protected DatabendConstant apply(DatabendConstant value) { + return value; } }, UNARY_MINUS("-", DatabendDataType.INT) { @Override - public DatabendDataType getExpressionType() { - return DatabendDataType.INT; + public DatabendDataType getExpressionType(DatabendExpression expr) { + return expr.getExpectedType(); } @Override - protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { - return null; + protected DatabendConstant apply(DatabendConstant value) { + if (value.isNull()) { + return DatabendConstant.createNullConstant(); + } + try { + if (value.isInt()) { + return DatabendConstant.createIntConstant(-value.asInt()); + } else if (value.isFloat()) { + return DatabendConstant.createFloatConstant(-value.asFloat()); + } else { + return null; + } + } catch (UnsupportedOperationException e) { + return null; + } } }; @@ -80,13 +96,13 @@ protected DatabendConstant getExpectedValue(DatabendConstant expectedValue) { this.dataTypes = dataTypes.clone(); } - public abstract DatabendDataType getExpressionType(); + public abstract DatabendDataType getExpressionType(DatabendExpression expr); public DatabendDataType getRandomInputDataTypes() { return Randomly.fromOptions(dataTypes); } - protected abstract DatabendConstant getExpectedValue(DatabendConstant expectedValue); + protected abstract DatabendConstant apply(DatabendConstant value); @Override public String getTextRepresentation() { diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index dcef8d7a8..e13ccdc7c 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -4,26 +4,27 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; public class DatabendDeleteGenerator { - public DatabendDeleteGenerator() { + private DatabendDeleteGenerator() { } - public static SQLQueryAdapter generate(DatabendGlobalState globalState){ + public static SQLQueryAdapter generate(DatabendGlobalState globalState) { StringBuilder sb = new StringBuilder("DELETE FROM "); ExpectedErrors errors = new ExpectedErrors(); sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); - if(Randomly.getBoolean()) { + if (Randomly.getBoolean()) { sb.append(" WHERE "); - sb.append(DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) - .generateExpression(DatabendDataType.BOOLEAN))); + sb.append(DatabendToStringVisitor.asString(DatabendExprToNode.cast( + new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN)))); DatabendErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(),errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 340e494ef..a86f0f3a9 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -8,6 +8,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; @@ -48,8 +49,9 @@ protected void insertValue(DatabendColumn column) { // } else { // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); // } - String value = DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) - .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 + String value = DatabendToStringVisitor + .asString(DatabendExprToNode.cast(new DatabendNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); // 生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index a67a5d4ff..7e2c48e00 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -2,50 +2,106 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendRowValue; +import sqlancer.databend.ast.DatabendAggregateOperation; +import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; +import sqlancer.databend.ast.DatabendBetweenOperation; import sqlancer.databend.ast.DatabendBinaryArithmeticOperation; import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; import sqlancer.databend.ast.DatabendBinaryComparisonOperation; import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; import sqlancer.databend.ast.DatabendBinaryLogicalOperation; import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; +import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendInOperation; import sqlancer.databend.ast.DatabendLikeOperation; +import sqlancer.databend.ast.DatabendOrderByTerm; import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; public class DatabendNewExpressionGenerator - extends TypedExpressionGenerator, DatabendColumn, DatabendDataType> { + extends TypedExpressionGenerator { private final DatabendGlobalState globalState; + + private final int maxDepth; private boolean allowAggregateFunctions; + private DatabendRowValue rowValue; + + private Set columnOfLeafNode; + + public DatabendNewExpressionGenerator setRowValue(DatabendRowValue rowValue) { + this.rowValue = rowValue; + return this; + } + + public void setColumnOfLeafNode(Set columnOfLeafNode) { + this.columnOfLeafNode = columnOfLeafNode; + } public DatabendNewExpressionGenerator(DatabendGlobalState globalState) { this.globalState = globalState; + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); } @Override - public Node generateLeafNode(DatabendDataType dataType) { - return generateConstant(dataType); + public DatabendExpression generateLeafNode(DatabendDataType dataType) { + if (Randomly.getBoolean()) { + return generateConstant(dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(dataType); + } else { + return createColumnOfType(dataType); + } + } + } + + final List filterColumns(DatabendDataType dataType) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType().getPrimitiveDataType() == dataType) + .collect(Collectors.toList()); + } + } + + private DatabendExpression createColumnOfType(DatabendDataType type) { + List columns = filterColumns(type); + DatabendColumn column = Randomly.fromList(columns); + DatabendConstant value = rowValue == null ? null : rowValue.getValues().get(column); + if (columnOfLeafNode != null) { + columnOfLeafNode.add(DatabendColumnValue.create(column, value)); + } + return DatabendColumnValue.create(column, value); + } + + public List> generateOrderBy() { + List randomColumns = Randomly.subset(columns); + return randomColumns.stream() + .map(c -> new DatabendOrderByTerm(new DatabendColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) + .collect(Collectors.toList()); } @Override - protected Node generateExpression(DatabendDataType type, int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + protected DatabendExpression generateExpression(DatabendDataType type, int depth) { + if (Randomly.getBooleanWithRatherLowProbability() || depth >= maxDepth) { return generateLeafNode(type); } @@ -63,8 +119,8 @@ protected Node generateExpression(DatabendDataType type, int } } - public List> generateExpressions(int nr, DatabendDataType type) { - List> expressions = new ArrayList<>(); + public List generateExpressions(int nr, DatabendDataType type) { + List expressions = new ArrayList<>(); for (int i = 0; i < nr; i++) { expressions.add(generateExpression(type)); } @@ -75,7 +131,7 @@ private enum IntExpression { UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION } - private Node generateIntExpression(int depth) { + private DatabendExpression generateIntExpression(int depth) { if (allowAggregateFunctions) { allowAggregateFunctions = false; } @@ -99,7 +155,7 @@ private enum BooleanExpression { // SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; } - Node generateBooleanExpression(int depth) { + DatabendExpression generateBooleanExpression(int depth) { if (allowAggregateFunctions) { allowAggregateFunctions = false; } @@ -126,40 +182,39 @@ Node generateBooleanExpression(int depth) { } - Node getPostfix(int depth) { + DatabendExpression getPostfix(int depth) { DatabendUnaryPostfixOperator randomOp = DatabendUnaryPostfixOperator.getRandom(); return new DatabendUnaryPostfixOperation( - generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp, - Randomly.getBoolean()); + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp); } - Node getNOT(int depth) { + DatabendExpression getNOT(int depth) { DatabendUnaryPrefixOperator op = DatabendUnaryPrefixOperator.NOT; return new DatabendUnaryPrefixOperation(generateExpression(op.getRandomInputDataTypes(), depth), op); } - Node getBetween(int depth) { + DatabendExpression getBetween(int depth) { // 跳过boolean DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); - return new NewBetweenOperatorNode(generateExpression(dataType, depth), - generateExpression(dataType, depth), generateExpression(dataType, depth), Randomly.getBoolean()); + return new DatabendBetweenOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + generateExpression(dataType, depth), Randomly.getBoolean()); } - Node getIn(int depth) { + DatabendExpression getIn(int depth) { DatabendDataType dataType = Randomly.fromOptions(DatabendDataType.values()); - Node leftExpr = generateExpression(dataType, depth); - List> rightExprs = new ArrayList<>(); + DatabendExpression leftExpr = generateExpression(dataType, depth); + List rightExprs = new ArrayList<>(); int nr = Randomly.smallNumber() + 1; for (int i = 0; i < nr; i++) { rightExprs.add(generateExpression(dataType, depth)); } - return new NewInOperatorNode(leftExpr, rightExprs, Randomly.getBoolean()); + return new DatabendInOperation(leftExpr, rightExprs, Randomly.getBoolean()); } - Node getBinaryLogical(int depth, DatabendDataType dataType) { - Node expr = generateExpression(dataType, depth); + DatabendExpression getBinaryLogical(int depth, DatabendDataType dataType) { + DatabendExpression expr = generateExpression(dataType, depth); int nr = Randomly.smallNumber() + 1; for (int i = 0; i < nr; i++) { expr = new DatabendBinaryLogicalOperation(expr, generateExpression(DatabendDataType.BOOLEAN, depth), @@ -168,37 +223,47 @@ Node getBinaryLogical(int depth, DatabendDataType dataType) return expr; } - Node getComparison(int depth) { + DatabendExpression getComparison(int depth) { // 跳过boolean DatabendDataType dataType = Randomly.fromList(Arrays.asList(DatabendDataType.values()).stream() .filter(t -> t != DatabendDataType.BOOLEAN).collect(Collectors.toList())); - Node leftExpr = generateExpression(dataType, depth); - Node rightExpr = generateExpression(dataType, depth); + DatabendExpression leftExpr = generateExpression(dataType, depth); + DatabendExpression rightExpr = generateExpression(dataType, depth); return new DatabendBinaryComparisonOperation(leftExpr, rightExpr, Randomly.fromOptions(DatabendBinaryComparisonOperator.values())); } - Node getLike(int depth, DatabendDataType dataType) { + DatabendExpression getLike(int depth, DatabendDataType dataType) { return new DatabendLikeOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), DatabendLikeOperation.DatabendLikeOperator.LIKE_OPERATOR); } + public DatabendExpression generateExpressionWithExpectedResult(DatabendDataType type) { + // DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + // gen.setRowValue(rowValue); + DatabendExpression expr; + do { + expr = this.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + @Override - public Node generatePredicate() { + public DatabendExpression generatePredicate() { return generateExpression(DatabendDataType.BOOLEAN); } @Override - public Node negatePredicate(Node predicate) { + public DatabendExpression negatePredicate(DatabendExpression predicate) { return new DatabendUnaryPrefixOperation(predicate, DatabendUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node predicate) { + public DatabendExpression isNull(DatabendExpression predicate) { return new DatabendUnaryPostfixOperation(predicate, DatabendUnaryPostfixOperator.IS_NULL); } - public Node generateConstant(DatabendDataType type, boolean isNullable) { + public DatabendExpression generateConstant(DatabendDataType type, boolean isNullable) { if (isNullable && Randomly.getBooleanWithSmallProbability()) { createConstant(DatabendDataType.NULL); } @@ -206,14 +271,14 @@ public Node generateConstant(DatabendDataType type, boolean } @Override - public Node generateConstant(DatabendDataType type) { + public DatabendExpression generateConstant(DatabendDataType type) { if (Randomly.getBooleanWithSmallProbability()) { return DatabendConstant.createNullConstant(); } return createConstant(type); } - public Node createConstant(DatabendDataType type) { + public DatabendExpression createConstant(DatabendDataType type) { Randomly r = globalState.getRandomly(); switch (type) { case INT: @@ -234,7 +299,7 @@ public Node createConstant(DatabendDataType type) { } @Override - protected Node generateColumn(DatabendDataType type) { + protected DatabendExpression generateColumn(DatabendDataType type) { return null; } @@ -248,52 +313,20 @@ protected boolean canGenerateColumnOfType(DatabendDataType type) { return false; } - public enum DatabendAggregateFunction { - MAX(1), MIN(1), AVG(1, DatabendDataType.INT, DatabendDataType.FLOAT), COUNT(1), - SUM(1, DatabendDataType.INT, DatabendDataType.FLOAT), STDDEV_POP(1), COVAR_POP(1), COVAR_SAMP(2); - //, *_IF, *_DISTINCT - - private int nrArgs; - private DatabendDataType[] dataTypes; - - DatabendAggregateFunction(int nrArgs, DatabendDataType... dataTypes) { - this.nrArgs = nrArgs; - this.dataTypes = dataTypes.clone(); - } - - public static DatabendAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public DatabendDataType getRandomType() { - if (dataTypes.length == 0) { - return Randomly.fromOptions(DatabendDataType.values()); - } else { - return Randomly.fromOptions(dataTypes); - } - } - - public int getNrArgs() { - return nrArgs; - } - - } - - public NewFunctionNode generateArgsForAggregate( - DatabendAggregateFunction aggregateFunction) { - return new NewFunctionNode( + public DatabendExpression generateArgsForAggregate(DatabendAggregateFunction aggregateFunction) { + return new DatabendAggregateOperation( generateExpressions(aggregateFunction.getNrArgs(), aggregateFunction.getRandomType()), aggregateFunction); } - public Node generateAggregate() { + public DatabendExpression generateAggregate() { DatabendAggregateFunction aggrFunc = DatabendAggregateFunction.getRandom(); return generateArgsForAggregate(aggrFunc); } - public Node generateHavingClause() { + public DatabendExpression generateHavingClause() { allowAggregateFunctions = true; - Node expression = generateExpression(DatabendDataType.BOOLEAN); + DatabendExpression expression = generateExpression(DatabendDataType.BOOLEAN); allowAggregateFunctions = false; return expression; } diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 70422b324..93f7b5262 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -1,18 +1,20 @@ package sqlancer.databend.gen; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -24,26 +26,29 @@ private DatabendRandomQuerySynthesizer() { } public static DatabendSelect generateSelect(DatabendGlobalState globalState, int nrColumns) { - DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + DatabendTables targetTables = globalState.getSchema().getRandomTableNonEmptyAndViewTables(); List targetColumns = targetTables.getColumns(); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(targetColumns); - DatabendSelect select = new DatabendSelect(); - // TODO distinct - select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); List> columns = new ArrayList<>(); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); int freeColumns = targetColumns.size(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = null; + Node column = null; if (freeColumns > 0 && Randomly.getBoolean()) { - expression = new ColumnReferenceNode<>(targetColumns.get(freeColumns - 1)); + column = new DatabendColumnValue(targetColumns.get(freeColumns - 1), null); freeColumns -= 1; + columnOfLeafNode.add((DatabendColumnValue) column); } else { - expression = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); + column = DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - columns.add(expression); + columns.add(column); } + DatabendSelect select = new DatabendSelect(); + boolean isDistinct = Randomly.getBoolean(); + select.setDistinct(isDistinct); select.setFetchColumns(columns); List tables = targetTables.getTables(); List> tableList = tables.stream() @@ -52,14 +57,28 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setJoinList(joins); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); + select.setWhereClause( + DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); } - if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + + List> noExprColumns = new ArrayList<>(columnOfLeafNode); + + if (Randomly.getBoolean() && noExprColumns.size() > 0) { + if (!isDistinct) { + select.setOrderByExpressions(Randomly.nonEmptySubset(noExprColumns)); + } + // TODO (for SELECT DISTINCT, ORDER BY expressions must appear in select list) isDistinct + // 需要orderby输入每个select list，可以用数字代替比如：1,2,3... } - if (Randomly.getBoolean()) { // TODO 该表达式生成的类型较多，需要对其限制 - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { // 可能产生新的column叶子结点 + select.setHavingClause(DatabendExprToNode.cast(gen.generateHavingClause())); + } + + noExprColumns = new ArrayList<>(columnOfLeafNode); + + if (Randomly.getBoolean() && noExprColumns.size() > 0) { + select.setGroupByExpressions(noExprColumns); } if (Randomly.getBoolean()) { @@ -71,10 +90,6 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int DatabendConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); } - if (Randomly.getBoolean()) { - select.setHavingClause(gen.generateHavingClause()); - } - return select; } diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 32505f98e..a34279fd2 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -4,10 +4,10 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -25,7 +25,7 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { sb.append(tableName); sb.append("("); List columns = getNewColumns(); - TypedExpressionGenerator, DatabendColumn, DatabendDataType> gen = new DatabendNewExpressionGenerator( + TypedExpressionGenerator gen = new DatabendNewExpressionGenerator( globalState).setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -45,7 +45,8 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 - gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); + DatabendExprToNode + .cast(gen.generateConstant(columns.get(i).getType().getPrimitiveDataType())))); sb.append(")"); } } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index c6d9c9229..e0c014f2b 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -18,6 +18,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -44,10 +45,14 @@ public DatabendNoRECOracle(DatabendGlobalState globalState) { @Override public void check() throws SQLException { - DatabendTables randomTables = s.getRandomTableNonEmptyTables(); // 随机获得nr张表 + DatabendTables randomTables = s.getRandomTableNonEmptyAndViewTables(); // 随机获得nr张表 List columns = randomTables.getColumns(); + if (columns.isEmpty()) { + debugColumns(columns, randomTables); // 调试代码，可忽略 + } DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件 + Node randomWhereCondition = DatabendExprToNode + .cast(gen.generateExpression(DatabendDataType.BOOLEAN)); // 生成随机where条件 List tables = randomTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -108,8 +113,8 @@ private int getOptimizedQueryCount(SQLConnection con, List columns, DatabendTables randomTables) { + DatabendTables test = new DatabendTables(s.getDatabaseTables()); + System.out.println(String.format("tables size: %d", test.getTables().size())); + for (DatabendTable table : test.getTables()) { + System.out.println(String.format("%s", table.getName())); + for (DatabendColumn column : table.getColumns()) { + System.out.println(String.format("%s %s", column.getName(), column.getType())); + } + System.out.println("------------------------"); + } + System.out.println("+++++++++++++++++++++++++++++"); + for (DatabendTable table : randomTables.getTables()) { + System.out.println(String.format("%s", table.getName())); + for (DatabendColumn column : table.getColumns()) { + System.out.println(String.format("%s %s", column.getName(), column.getType())); + } + System.out.println("------------------------"); + } + throw new AssertionError( + String.format("randomTables size: %d,column is empty", randomTables.getTables().size())); + } + } diff --git a/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..bf0bf6bea --- /dev/null +++ b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java @@ -0,0 +1,154 @@ +package sqlancer.databend.test; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExpectedValueVisitor; +import sqlancer.databend.DatabendExprToNode; +import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendDataType; +import sqlancer.databend.DatabendSchema.DatabendRowValue; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendColumnValue; +import sqlancer.databend.ast.DatabendConstant; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; + +public class DatabendPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + + public DatabendPivotedQuerySynthesisOracle(DatabendGlobalState globalState) { + super(globalState); + DatabendErrors.addExpressionErrors(errors); + DatabendErrors.addInsertErrors(errors); + } + + @Override + protected Query getRectifiedQuery() throws Exception { + DatabendTables randomTables = globalState.getSchema().getRandomTableNonEmptyAndViewTables(); + List columns = randomTables.getColumns(); + DatabendSelect selectStatement = new DatabendSelect(); + boolean isDistinct = Randomly.getBoolean(); + selectStatement.setDistinct(isDistinct); + pivotRow = randomTables.getRandomRowValue(globalState.getConnection()); + fetchColumns = columns; + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new DatabendColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + selectStatement.setFromList(randomTables.getTables().stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList())); + DatabendExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(DatabendExprToNode.cast(whereClause)); + List> groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + Node limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + Node offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + if (!isDistinct) { + List> orderBys = gen.generateOrderBy(); + selectStatement.setOrderByExpressions(orderBys); + } + return new SQLQueryAdapter(DatabendToStringVisitor.asString(selectStatement), errors); + } + + private DatabendExpression generateRectifiedExpression(List columns, DatabendRowValue pivotRow) { + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + gen.setRowValue(pivotRow); + DatabendExpression expr = gen.generateExpressionWithExpectedResult(DatabendDataType.BOOLEAN); + DatabendExpression result = null; + if (expr.getExpectedValue().isNull()) { + result = new DatabendUnaryPostfixOperation(expr, + DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator.IS_NULL); + } else if (!expr.getExpectedValue().cast(DatabendDataType.BOOLEAN).asBoolean()) { + result = new DatabendUnaryPrefixOperation(expr, + DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator.NOT); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); + sb.append(pivotRowQuery.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (DatabendColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL "); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).toString()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + private DatabendColumn getFetchValueAliasedColumn(DatabendColumn c) { + DatabendColumn aliasedColumn = new DatabendColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType(), false, false); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + @Override + protected String getExpectedValues(DatabendExpression expr) { + return DatabendExpectedValueVisitor.asExpectedValues(DatabendExprToNode.cast(expr)); + } + + private List> generateGroupByClause(List columns, + DatabendRowValue rowValue) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> new DatabendColumnValue(c, rowValue.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private Node generateLimit() { + if (Randomly.getBoolean()) { + return DatabendConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private Node generateOffset() { + if (Randomly.getBoolean()) { + return DatabendConstant.createIntConstant(0); + } else { + return null; + } + } + +} diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java similarity index 91% rename from src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 56f311f2b..ca591971f 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -18,17 +18,19 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendAggregateOperation; +import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; import sqlancer.databend.ast.DatabendCastOperation; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; -import sqlancer.databend.gen.DatabendNewExpressionGenerator.DatabendAggregateFunction; public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase implements TestOracle { @@ -48,12 +50,12 @@ public void check() throws SQLException { DatabendAggregateFunction aggregateFunction = Randomly.fromOptions(DatabendAggregateFunction.MAX, DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT, DatabendAggregateFunction.AVG/* , DatabendAggregateFunction.STDDEV_POP */); - NewFunctionNode aggregate = gen + NewFunctionNode aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); List> fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { - fetchColumns.add(gen.generateAggregate()); // TODO 更换成非聚合函数 + fetchColumns.add((DatabendAggregateOperation) gen.generateAggregate()); // TODO 更换成非聚合函数 } select.setFetchColumns(Arrays.asList(aggregate)); // if (Randomly.getBooleanWithRatherLowProbability()) { @@ -81,7 +83,8 @@ private String createMetamorphicUnionQuery(DatabendSelect select, NewFunctionNode aggregate, List> from) { String metamorphicQuery; - Node whereClause = gen.generateExpression(DatabendDataType.BOOLEAN); + Node whereClause = DatabendExprToNode + .cast(gen.generateExpression(DatabendDataType.BOOLEAN)); Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, DatabendUnaryPrefixOperator.NOT); Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, @@ -186,9 +189,10 @@ private DatabendSelect getSelect(List> aggregates, List select.setFromList(from); select.setWhereClause(whereClause); select.setJoinList(joinList); - if (Randomly.getBooleanWithSmallProbability()) { - select.setGroupByExpressions(List.of(gen.generateConstant(DatabendDataType.INT))); // TODO 仍可加强 - } + if (Randomly.getBooleanWithSmallProbability()) { + select.setGroupByExpressions(List.of(DatabendExprToNode.cast(gen.generateConstant(DatabendDataType.INT)))); // TODO + // 仍可加强 + } return select; } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java similarity index 73% rename from src/sqlancer/databend/test/DatabendQueryPartitioningBase.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 6ef132572..c836fad02 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -1,7 +1,8 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.stream.Collectors; @@ -19,19 +20,22 @@ import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendQueryPartitioningBase extends - TernaryLogicPartitioningOracleBase, DatabendGlobalState> implements TestOracle { +public class DatabendQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase implements TestOracle { DatabendSchema s; DatabendTables targetTables; DatabendNewExpressionGenerator gen; DatabendSelect select; + List> groupByExpression; + public DatabendQueryPartitioningBase(DatabendGlobalState state) { super(state); DatabendErrors.addExpressionErrors(errors); @@ -49,11 +53,20 @@ public static String canonicalizeResultValue(String value) { @Override public void check() throws SQLException { s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); + targetTables = s.getRandomTableNonEmptyAndViewTables(); + List randomColumn = targetTables.getColumns(); + gen = new DatabendNewExpressionGenerator(state).setColumns(targetTables.getColumns()); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); initializeTernaryPredicateVariants(); select = new DatabendSelect(); - select.setFetchColumns(generateRandomColumns()); + columnOfLeafNode + .addAll(randomColumn.stream().map(c -> new DatabendColumnValue(c, null)).collect(Collectors.toList())); + groupByExpression = new ArrayList<>(columnOfLeafNode); + + select.setFetchColumns(randomColumn.stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList())); List tables = targetTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); @@ -65,7 +78,7 @@ public void check() throws SQLException { List> generateFetchColumns() { List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { // TODO 为什么会返回 false 或 true 字段 + if (Randomly.getBoolean()) { columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); } else { columns = generateRandomColumns(); @@ -81,7 +94,7 @@ List> generateRandomColumns() { } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java similarity index 78% rename from src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java index 8eae1e502..cb46e9196 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -7,7 +7,9 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; +import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; public class DatabendQueryPartitioningDistinctTester extends DatabendQueryPartitioningBase { @@ -21,18 +23,18 @@ public DatabendQueryPartitioningDistinctTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); select.setDistinct(true); - select.setWhereClause(null); + select.setWhereClause(DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); String originalQueryString = DatabendToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); if (Randomly.getBoolean()) { select.setDistinct(false); } - select.setWhereClause(predicate); + select.setWhereClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); + select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); + select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java similarity index 85% rename from src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java index dd25b6d7f..cbac6820b 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningGroupByTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -10,6 +10,7 @@ import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendToStringVisitor; @@ -25,17 +26,17 @@ public DatabendQueryPartitioningGroupByTester(DatabendGlobalState state) { @Override public void check() throws SQLException { super.check(); - select.setGroupByExpressions(select.getFetchColumns()); + select.setGroupByExpressions(groupByExpression); select.setWhereClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setWhereClause(predicate); + select.setWhereClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); + select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); + select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java similarity index 80% rename from src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java index 8ca1489f2..419c2d558 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -10,6 +10,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; @@ -27,7 +28,8 @@ public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); + select.setWhereClause( + DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); } // boolean orderBy = Randomly.getBoolean(); boolean orderBy = false; // 关闭order by @@ -38,18 +40,16 @@ public void check() throws SQLException { Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByExpressions(constants); } - // select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - select.setGroupByExpressions(select.getFetchColumns()); + select.setGroupByExpressions(groupByExpression); select.setHavingClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); - // System.out.println(originalQueryString); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setHavingClause(predicate); + select.setHavingClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setHavingClause(negatedPredicate); + select.setHavingClause(DatabendExprToNode.cast(negatedPredicate)); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setHavingClause(isNullPredicate); + select.setHavingClause(DatabendExprToNode.cast(isNullPredicate)); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, @@ -59,13 +59,13 @@ public void check() throws SQLException { } @Override - protected Node generatePredicate() { + protected DatabendExpression generatePredicate() { return gen.generateHavingClause(); } @Override List> generateFetchColumns() { - return Collections.singletonList(gen.generateHavingClause()); + return Collections.singletonList(DatabendExprToNode.cast(gen.generateHavingClause())); } } diff --git a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java similarity index 85% rename from src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java rename to src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index 2b43d0ba0..cff5ff1dd 100644 --- a/src/sqlancer/databend/test/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -1,4 +1,4 @@ -package sqlancer.databend.test; +package sqlancer.databend.test.tlp; import java.sql.SQLException; import java.util.ArrayList; @@ -6,6 +6,7 @@ import sqlancer.ComparatorHelper; import sqlancer.databend.DatabendErrors; +import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendToStringVisitor; @@ -29,11 +30,11 @@ public void check() throws SQLException { // if (orderBy) { //TODO 待开启 // select.setOrderByExpressions(gen.generateOrderBys()); // } - select.setWhereClause(predicate); + select.setWhereClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); + select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); + select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index 0c1755a54..ae1ff1d88 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -33,6 +33,18 @@ public void testDatabendTLPQueryPartitioning() { "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); } + @Test + public void testDatabendPQS() { + String databendAvailable = System.getenv("DATABEND_AVAILABLE"); + boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); + assumeTrue(databendIsAvailable); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); + } + // @Test // public void testDatabendTLPWhere() { // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); @@ -41,7 +53,7 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "WHERE")); // } // @@ -53,7 +65,7 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "GROUP_BY")); // } // @@ -65,7 +77,7 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "HAVING")); // } // @@ -77,7 +89,7 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "DISTINCT")); // } // @@ -89,18 +101,8 @@ public void testDatabendTLPQueryPartitioning() { // assertEquals(0, // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), + // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "AGGREGATE")); // } - // @Test - // void testConnection() { - // assertEquals(0, - // Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - // "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.NUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "--username", "user1", "--password", "1234", - // "databend", "--oracle", "HAVING" })); - // } - } From 01f4c67c9f2f001594b520997fab2fec51e9131f Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 12 Dec 2022 08:55:47 +0800 Subject: [PATCH 254/856] tidb: add BIT_AND and BIT_OR (#634) * tidb: add BIT_AND and BIT_OR Signed-off-by: Weizhen Wang * fmt * fmt * fmt Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/ast/TiDBAggregate.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sqlancer/tidb/ast/TiDBAggregate.java b/src/sqlancer/tidb/ast/TiDBAggregate.java index 21bd8eb0c..6650dc395 100644 --- a/src/sqlancer/tidb/ast/TiDBAggregate.java +++ b/src/sqlancer/tidb/ast/TiDBAggregate.java @@ -9,11 +9,7 @@ public class TiDBAggregate extends FunctionNode implements TiDBExpression { public enum TiDBAggregateFunction { - COUNT(1), // - SUM(1), // - AVG(1), // - MIN(1), // - MAX(1); + AVG(1), BIT_AND(1), BIT_OR(1), COUNT(1), SUM(1), MIN(1), MAX(1); private int nrArgs; From e2e037177ad50e50187adea2e29982428477021d Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Tue, 13 Dec 2022 15:11:16 +0800 Subject: [PATCH 255/856] fix(databend): distinct tester Signed-off-by: Yisong Han --- .../test/tlp/DatabendQueryPartitioningDistinctTester.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java index cb46e9196..5756a00be 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java @@ -9,7 +9,6 @@ import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; public class DatabendQueryPartitioningDistinctTester extends DatabendQueryPartitioningBase { @@ -23,7 +22,9 @@ public DatabendQueryPartitioningDistinctTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); select.setDistinct(true); - select.setWhereClause(DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + // TODO 后期可以使用and来进行扩展 + // select.setWhereClause(DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + select.setWhereClause(null); String originalQueryString = DatabendToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); From a4dc79714435b8b9e226b0b299669d9a5874d6d0 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 15 Dec 2022 22:42:29 +0800 Subject: [PATCH 256/856] tidb: support HASH_JOIN_BUILD and HASH_JOIN_PROBE (#639) * tidb: support HASH_JOIN_BUILD and HASH_JOIN_PROBE Signed-off-by: Weizhen Wang * tidb: support HASH_JOIN_BUILD and HASH_JOIN_PROBE Signed-off-by: Weizhen Wang * tidb: support HASH_JOIN_BUILD and HASH_JOIN_PROBE Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBHintGenerator.java | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index f72eafe16..64f8540bb 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -31,7 +31,9 @@ enum IndexHint { // READ_FROM_STORAGE USE_INDEX_MERGE, // NO_INDEX_MERGE, // - USE_TOJA; + USE_TOJA, // + HASH_JOIN_BUILD, // + HASH_JOIN_PROBE; } public TiDBHintGenerator(TiDBSelect select, List tables) { @@ -100,6 +102,16 @@ private void generate() { sb.append(Randomly.getBoolean()); sb.append(")"); break; + case HASH_JOIN_BUILD: + sb.append("HASH_JOIN_BUILD("); + sb.append(table.getName()); + sb.append(")"); + break; + case HASH_JOIN_PROBE: + sb.append("HASH_JOIN_PROBE("); + sb.append(table.getName()); + sb.append(")"); + break; default: throw new AssertionError(); } From dfcb0feeafbee11c3e133d8b4cf61a4d955bf389 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Fri, 16 Dec 2022 11:14:43 +0800 Subject: [PATCH 257/856] feat(databend):add DatabendBugs Signed-off-by: Yisong Han --- src/sqlancer/databend/DatabendBugs.java | 15 +++++++++++++ src/sqlancer/databend/DatabendErrors.java | 21 ++++++++++++++++--- .../tlp/DatabendQueryPartitioningBase.java | 7 +++++-- 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 src/sqlancer/databend/DatabendBugs.java diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java new file mode 100644 index 000000000..9a8f788c7 --- /dev/null +++ b/src/sqlancer/databend/DatabendBugs.java @@ -0,0 +1,15 @@ +package sqlancer.databend; + +public final class DatabendBugs { + + public static boolean bug9162; // https://github.com/datafuselabs/databend/issues/9162 + public static boolean bug9163; // https://github.com/datafuselabs/databend/issues/9163 + public static boolean bug9018; // https://github.com/datafuselabs/databend/issues/9018 + public static boolean bug9224 = true; // https://github.com/datafuselabs/databend/issues/9224 + public static boolean bug9234 = true; // https://github.com/datafuselabs/databend/issues/9234 + public static boolean bug9236 = true; // https://github.com/datafuselabs/databend/issues/9236 + public static boolean bug9264 = true; // https://github.com/datafuselabs/databend/issues/9264 + + private DatabendBugs() { + } +} diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 46202830b..6f061d901 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -12,10 +12,25 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("/ by zero"); errors.add("ORDER BY position"); errors.add("GROUP BY position"); - errors.add("downcast column error"); // bug - errors.add("index out of bounds"); // bug + if (DatabendBugs.bug9162) { + errors.add("downcast column error"); + } + if (DatabendBugs.bug9018) { + errors.add("index out of bounds"); + } + if (DatabendBugs.bug9163) { + errors.add("validity must be equal to the array's length"); + } + if (DatabendBugs.bug9224) { + errors.add("Can't cast column from nullable data into non-nullable type"); + } + if (DatabendBugs.bug9234) { + errors.add("called `Option::unwrap()` on a `None` value"); + } + if (DatabendBugs.bug9264) { + errors.add("assertion failed: offset + length <= self.length"); + } errors.add("validity's length must be equal"); // bug - errors.add("validity must be equal to the array's length"); // bug /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 74257310e..28d516f65 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -14,6 +14,7 @@ import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; @@ -71,8 +72,10 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DatabendJoin.getJoins(tableList, state); - select.setJoinList(joins.stream().collect(Collectors.toList())); + if (!DatabendBugs.bug9236) { + List> joins = DatabendJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + } select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } From cec99caa91dbcb5dcb0176fbabbd26d0e5e8a8ac Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Fri, 16 Dec 2022 16:16:31 +0800 Subject: [PATCH 258/856] Escaping newlines --- src/sqlancer/common/log/SQLLoggableFactory.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java index 24a28a603..790d10709 100644 --- a/src/sqlancer/common/log/SQLLoggableFactory.java +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -14,6 +14,8 @@ protected Loggable createLoggable(String input, String suffix) { if (!input.endsWith(";")) { completeString += ";"; } + completeString = completeString.replace("\n", "\\n"); + completeString = completeString.replace("\r", "\\r"); if (suffix != null && suffix.length() != 0) { completeString += suffix; } From f3f44c313fb7eb3936edc612f5241c806a03658b Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Fri, 16 Dec 2022 16:50:30 +0800 Subject: [PATCH 259/856] fix bug in CREATE TABLE --- src/sqlancer/common/query/SQLQueryAdapter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 01fcc8539..ab9124230 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -46,7 +46,7 @@ private String canonicalizeString(String s) { } private void checkQueryString() { - if (query.contains("CREATE TABLE") && !couldAffectSchema) { + if (query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN") && !couldAffectSchema) { throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); } } From f65cadd2ee3d9c61cd4ed8fb1a32fcb5287cfb91 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Fri, 16 Dec 2022 21:47:03 +0800 Subject: [PATCH 260/856] random virtual/rtree table names generation (#647) --- .../sqlite3/schema/SQLite3Schema.java | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index b4bf11002..586d7e776 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -462,4 +462,32 @@ public List getDatabaseTablesWithoutViewsWithoutVirtualTables() { return getDatabaseTables().stream().filter(t -> !t.isView() && !t.isVirtual).collect(Collectors.toList()); } + public String getFreeVirtualTableName() { + int i = 0; + if (Randomly.getBooleanWithRatherLowProbability()) { + i = (int) Randomly.getNotCachedInteger(0, 100); + } + do { + String tableName = String.format("vt%d", i++); + if (getDatabaseTables().stream().noneMatch(t -> t.getName().equalsIgnoreCase(tableName))) { + return tableName; + } + } while (true); + + } + + public String getFreeRtreeTableName() { + int i = 0; + if (Randomly.getBooleanWithRatherLowProbability()) { + i = (int) Randomly.getNotCachedInteger(0, 100); + } + do { + String tableName = String.format("rt%d", i++); + if (getDatabaseTables().stream().noneMatch(t -> t.getName().equalsIgnoreCase(tableName))) { + return tableName; + } + } while (true); + + } + } From a3357c13efba0724435724dec0147ec71c27c5e7 Mon Sep 17 00:00:00 2001 From: Yisong Han <1149869798@qq.com> Date: Fri, 16 Dec 2022 22:00:34 +0800 Subject: [PATCH 261/856] add(databend): databend bugs (#646) Signed-off-by: Yisong Han Signed-off-by: Yisong Han --- src/sqlancer/databend/DatabendBugs.java | 7 ++++++- src/sqlancer/databend/DatabendErrors.java | 2 +- src/sqlancer/databend/DatabendSchema.java | 3 ++- test/sqlancer/dbms/TestDatabend.java | 8 ++++---- 4 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index 9a8f788c7..336443511 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -2,11 +2,16 @@ public final class DatabendBugs { + public static boolean bug9018; // https://github.com/datafuselabs/databend/issues/9018 public static boolean bug9162; // https://github.com/datafuselabs/databend/issues/9162 public static boolean bug9163; // https://github.com/datafuselabs/databend/issues/9163 - public static boolean bug9018; // https://github.com/datafuselabs/databend/issues/9018 + public static boolean bug9164 = true; // https://github.com/datafuselabs/databend/issues/9164 + public static boolean bug9196 = true; // https://github.com/datafuselabs/databend/issues/9196 + public static boolean bug9232 = true; // https://github.com/datafuselabs/databend/issues/9232 public static boolean bug9224 = true; // https://github.com/datafuselabs/databend/issues/9224 + public static boolean bug9226 = true; // https://github.com/datafuselabs/databend/issues/9226 public static boolean bug9234 = true; // https://github.com/datafuselabs/databend/issues/9234 + public static boolean bug9235 = true; // https://github.com/datafuselabs/databend/issues/9235 public static boolean bug9236 = true; // https://github.com/datafuselabs/databend/issues/9236 public static boolean bug9264 = true; // https://github.com/datafuselabs/databend/issues/9264 diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 6f061d901..8f9d2b942 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -30,7 +30,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (DatabendBugs.bug9264) { errors.add("assertion failed: offset + length <= self.length"); } - errors.add("validity's length must be equal"); // bug + // errors.add("validity's length must be equal"); // bug /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index c82ad82ca..24d47ec46 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -173,7 +173,8 @@ public DatabendRowValue getRandomRowValue(SQLConnection con) throws SQLException try (Statement s = con.createStatement()) { ResultSet rs = s.executeQuery(rowValueQuery); if (!rs.next()) { - throw new AssertionError("could not find random row " + rowValueQuery + "\n"); + throw new IgnoreMeException(); + // throw new AssertionError("could not find random row " + rowValueQuery + "\n"); } for (int i = 0; i < getColumns().size(); i++) { DatabendColumn column = getColumns().get(i); diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java index ae1ff1d88..707d7b8a9 100644 --- a/test/sqlancer/dbms/TestDatabend.java +++ b/test/sqlancer/dbms/TestDatabend.java @@ -22,7 +22,7 @@ public void testDatabendNoREC() { } @Test - public void testDatabendTLPQueryPartitioning() { + public void testDatabendPQS() { String databendAvailable = System.getenv("DATABEND_AVAILABLE"); boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); assumeTrue(databendIsAvailable); @@ -30,11 +30,11 @@ public void testDatabendTLPQueryPartitioning() { Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); } @Test - public void testDatabendPQS() { + public void testDatabendTLPQueryPartitioning() { String databendAvailable = System.getenv("DATABEND_AVAILABLE"); boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); assumeTrue(databendIsAvailable); @@ -42,7 +42,7 @@ public void testDatabendPQS() { Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); } // @Test From 706cff79edab7df2282b975caff7495ef1f6f2f3 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 17 Dec 2022 04:41:46 +0800 Subject: [PATCH 262/856] tidb: add more hint (#649) * tidb: add SHUFFLE_JOIN, BROADCAST_JOIN, LIMIT_TO_COP, MPP_1PHASE_AGG, MPP_2PHASE_AGG Signed-off-by: Weizhen Wang * tidb: add SHUFFLE_JOIN, BROADCAST_JOIN, LIMIT_TO_COP, MPP_1PHASE_AGG, MPP_2PHASE_AGG Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBHintGenerator.java | 55 ++++++++++++++------ 1 file changed, 38 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index 64f8540bb..355cb2934 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -33,7 +33,12 @@ enum IndexHint { NO_INDEX_MERGE, // USE_TOJA, // HASH_JOIN_BUILD, // - HASH_JOIN_PROBE; + HASH_JOIN_PROBE, // + MPP_1PHASE_AGG, // + MPP_2PHASE_AGG, // + LIMIT_TO_COP, // + SHUFFLE_JOIN, // + BROADCAST_JOIN } public TiDBHintGenerator(TiDBSelect select, List tables) { @@ -73,6 +78,15 @@ private void generate() { case STREAM_AGG: sb.append("STREAM_AGG()"); break; + case MPP_1PHASE_AGG: + sb.append("MPP_1PHASE_AGG()"); + break; + case MPP_2PHASE_AGG: + sb.append("MPP_2PHASE_AGG()"); + break; + case LIMIT_TO_COP: + sb.append("LIMIT_TO_COP()"); + break; case USE_INDEX: indexesHint("USE_INDEX"); break; @@ -82,17 +96,11 @@ private void generate() { case AGG_TO_COP: sb.append("AGG_TO_COP()"); break; + case SHUFFLE_JOIN: + twoTablesHint("SHUFFLE_JOIN", table); + break; case USE_INDEX_MERGE: - if (table.hasIndexes()) { - sb.append("USE_INDEX_MERGE("); - sb.append(table.getName()); - sb.append(", "); - List indexes = Randomly.nonEmptySubset(table.getIndexes()); - sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining(", "))); - sb.append(")"); - } else { - throw new IgnoreMeException(); - } + twoTablesHint("USE_INDEX_MERGE", table); break; case NO_INDEX_MERGE: sb.append("NO_INDEX_MERGE()"); @@ -103,14 +111,13 @@ private void generate() { sb.append(")"); break; case HASH_JOIN_BUILD: - sb.append("HASH_JOIN_BUILD("); - sb.append(table.getName()); - sb.append(")"); + tablesHint("HASH_JOIN_BUILD"); break; case HASH_JOIN_PROBE: - sb.append("HASH_JOIN_PROBE("); - sb.append(table.getName()); - sb.append(")"); + tablesHint("HASH_JOIN_PROBE"); + break; + case BROADCAST_JOIN: + twoTablesHint("BROADCAST_JOIN", table); break; default: throw new AssertionError(); @@ -141,6 +148,20 @@ private void tablesHint(String string) { sb.append(")"); } + private void twoTablesHint(String string, TiDBTable table) { + if (table.hasIndexes()) { + sb.append(string); + sb.append("("); + sb.append(table.getName()); + sb.append(", "); + List indexes = Randomly.nonEmptySubset(table.getIndexes()); + sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining(", "))); + sb.append(")"); + } else { + throw new IgnoreMeException(); + } + } + private void appendTables() { List tableSubset = Randomly.nonEmptySubset(tables); sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); From 45cd972c90e0b4dafe5c126a795907fbcccaa265 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Dec 2022 21:00:47 +0000 Subject: [PATCH 263/856] build(deps): bump spotbugs-maven-plugin from 4.7.2.1 to 4.7.3.0 (#608) Bumps [spotbugs-maven-plugin](https://github.com/spotbugs/spotbugs-maven-plugin) from 4.7.2.1 to 4.7.3.0. - [Release notes](https://github.com/spotbugs/spotbugs-maven-plugin/releases) - [Commits](https://github.com/spotbugs/spotbugs-maven-plugin/compare/spotbugs-maven-plugin-4.7.2.1...spotbugs-maven-plugin-4.7.3.0) --- updated-dependencies: - dependency-name: com.github.spotbugs:spotbugs-maven-plugin dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 950ea4243..f0afd61c0 100644 --- a/pom.xml +++ b/pom.xml @@ -252,7 +252,7 @@ com.github.spotbugs spotbugs-maven-plugin - 4.7.2.1 + 4.7.3.0 spotbugs From 4c88763d64a98af4fc7bdf1d9b23afcb672b9166 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Dec 2022 21:26:30 +0000 Subject: [PATCH 264/856] build(deps): bump mariadb-java-client from 3.0.8 to 3.1.0 (#614) Bumps [mariadb-java-client](https://github.com/mariadb-corporation/mariadb-connector-j) from 3.0.8 to 3.1.0. - [Release notes](https://github.com/mariadb-corporation/mariadb-connector-j/releases) - [Changelog](https://github.com/mariadb-corporation/mariadb-connector-j/blob/master/CHANGELOG.md) - [Commits](https://github.com/mariadb-corporation/mariadb-connector-j/commits) --- updated-dependencies: - dependency-name: org.mariadb.jdbc:mariadb-java-client dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f0afd61c0..573e17c1e 100644 --- a/pom.xml +++ b/pom.xml @@ -309,7 +309,7 @@ org.mariadb.jdbc mariadb-java-client - 3.0.8 + 3.1.0 org.duckdb From fcb2b13b398ff62c5ef6b4e8d37ed9be51cf0fa2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 16 Dec 2022 22:15:05 +0000 Subject: [PATCH 265/856] build(deps): bump postgresql from 42.5.0 to 42.5.1 (#620) Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.5.0 to 42.5.1. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.5.0...REL42.5.1) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 573e17c1e..1ddb71c02 100644 --- a/pom.xml +++ b/pom.xml @@ -284,7 +284,7 @@ org.postgresql postgresql - 42.5.0 + 42.5.1 com.ing.data From b5eb2153dfd06bf50aca0f98eb32383b389a933f Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 16:59:34 +0800 Subject: [PATCH 266/856] Add interface to return queries by oracles (#648) Co-authored-by: Manuel Rigger --- .../cockroachdb/oracle/CockroachDBNoRECOracle.java | 5 +++++ .../oracle/tlp/CockroachDBTLPAggregateOracle.java | 5 +++++ .../oracle/tlp/CockroachDBTLPDistinctOracle.java | 9 ++++++++- .../oracle/tlp/CockroachDBTLPExtendedWhereOracle.java | 7 +++++++ .../oracle/tlp/CockroachDBTLPGroupByOracle.java | 9 ++++++++- .../oracle/tlp/CockroachDBTLPHavingOracle.java | 8 ++++++++ .../oracle/tlp/CockroachDBTLPWhereOracle.java | 9 ++++++++- src/sqlancer/common/oracle/CompositeTestOracle.java | 7 +++++++ src/sqlancer/common/oracle/TestOracle.java | 4 ++++ src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java | 5 +++++ .../sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java | 8 +++++++- .../sqlite3/oracle/tlp/SQLite3TLPDistinctOracle.java | 9 ++++++++- .../sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java | 9 ++++++++- .../sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java | 8 +++++++- .../sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java | 9 ++++++++- src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java | 8 ++++++++ src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java | 9 ++++++++- 17 files changed, 119 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 77ecbcbed..6ab133b1c 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -150,4 +150,9 @@ private int getCount(SQLGlobalState globalState, SQLQueryAdapter q) throws return count; } + @Override + public String getLastQueryString() { + return optimizedQueryString; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index 0368f5222..3cf614cea 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -195,4 +195,9 @@ private CockroachDBSelect getSelect(List aggregates, List return leftSelect; } + @Override + public String getLastQueryString() { + return originalQuery; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java index bdf8f59c3..dd174e805 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPDistinctOracle.java @@ -15,6 +15,8 @@ public class CockroachDBTLPDistinctOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPDistinctOracle(CockroachDBGlobalState state) { super(state); errors.add("GROUP BY term out of range"); @@ -25,7 +27,7 @@ public void check() throws SQLException { super.check(); select.setDistinct(true); String originalQueryString = CockroachDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setDistinct(false); CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); @@ -41,4 +43,9 @@ public void check() throws SQLException { ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java index 36ac46733..7463e767f 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java @@ -19,6 +19,7 @@ public class CockroachDBTLPExtendedWhereOracle extends CockroachDBTLPBase { private CockroachDBExpression originalPredicate; + private String generatedQueryString; public CockroachDBTLPExtendedWhereOracle(CockroachDBGlobalState state) { super(state); @@ -32,6 +33,7 @@ public void check() throws SQLException { originalPredicate = generatePredicate(); select.setWhereClause(originalPredicate); String originalQueryString = CockroachDBVisitor.asString(select); + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean allowOrderBy = Randomly.getBoolean(); @@ -56,4 +58,9 @@ public CockroachDBExpression combinePredicate(CockroachDBExpression expr) { return new CockroachDBBinaryLogicalOperation(originalPredicate, expr, CockroachDBBinaryLogicalOperator.AND); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java index 535fea3e1..466252e56 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPGroupByOracle.java @@ -14,6 +14,8 @@ public class CockroachDBTLPGroupByOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPGroupByOracle(CockroachDBGlobalState state) { super(state); } @@ -24,7 +26,7 @@ public void check() throws SQLException { select.setGroupByExpressions(select.getFetchColumns()); select.setWhereClause(null); String originalQueryString = CockroachDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setWhereClause(predicate); @@ -46,4 +48,9 @@ List generateFetchColumns() { .collect(Collectors.toList())); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java index 42abad00b..d5a96d80b 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java @@ -13,6 +13,8 @@ public class CockroachDBTLPHavingOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPHavingOracle(CockroachDBGlobalState state) { super(state); errors.add("GROUP BY term out of range"); @@ -31,6 +33,7 @@ public void check() throws SQLException { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); String originalQueryString = CockroachDBVisitor.asString(select); + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); @@ -52,4 +55,9 @@ protected CockroachDBExpression generatePredicate() { return gen.generateHavingClause(); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java index 8fe0899d0..db0ea6162 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java @@ -16,6 +16,8 @@ public class CockroachDBTLPWhereOracle extends CockroachDBTLPBase { + private String generatedQueryString; + public CockroachDBTLPWhereOracle(CockroachDBGlobalState state) { super(state); errors.add("GROUP BY term out of range"); @@ -25,7 +27,7 @@ public CockroachDBTLPWhereOracle(CockroachDBGlobalState state) { public void check() throws SQLException { super.check(); String originalQueryString = CockroachDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean allowOrderBy = Randomly.getBoolean(); @@ -45,4 +47,9 @@ public void check() throws SQLException { ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/common/oracle/CompositeTestOracle.java b/src/sqlancer/common/oracle/CompositeTestOracle.java index 0fdc13140..e2f96f785 100644 --- a/src/sqlancer/common/oracle/CompositeTestOracle.java +++ b/src/sqlancer/common/oracle/CompositeTestOracle.java @@ -9,6 +9,7 @@ public class CompositeTestOracle> implements Test private final List> oracles; private final G globalState; private int i; + private int iLast; public CompositeTestOracle(List> oracles, G globalState) { this.globalState = globalState; @@ -19,6 +20,7 @@ public CompositeTestOracle(List> oracles, G globalState) { public void check() throws Exception { try { oracles.get(i).check(); + iLast = i; boolean lastOracleIndex = i == oracles.size() - 1; if (!lastOracleIndex) { globalState.getManager().incrementSelectQueryCount(); @@ -27,4 +29,9 @@ public void check() throws Exception { i = (i + 1) % oracles.size(); } } + + @Override + public String getLastQueryString() { + return oracles.get(iLast).getLastQueryString(); + } } diff --git a/src/sqlancer/common/oracle/TestOracle.java b/src/sqlancer/common/oracle/TestOracle.java index 2dfd854a0..0ef993b89 100644 --- a/src/sqlancer/common/oracle/TestOracle.java +++ b/src/sqlancer/common/oracle/TestOracle.java @@ -10,4 +10,8 @@ public interface TestOracle> { default Reproducer getLastReproducer() { return null; } + + default String getLastQueryString() { + throw new AssertionError("Not supported!"); + } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 826a2727b..3a5a5c9d0 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -102,6 +102,11 @@ public Reproducer getLastReproducer() { return reproducer; } + @Override + public String getLastQueryString() { + return optimizedQueryString; + } + private Function getUnoptimizedQuery(SQLite3Select select, SQLite3Expression randomWhereCondition) throws SQLException { SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 715770257..90ca0d96b 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -33,6 +33,7 @@ public class SQLite3TLPAggregateOracle implements TestOracle private final SQLite3GlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); private SQLite3ExpressionGenerator gen; + private String generatedQueryString; public SQLite3TLPAggregateOracle(SQLite3GlobalState state) { this.state = state; @@ -56,7 +57,7 @@ public void check() throws SQLException { select.setOrderByExpressions(gen.generateOrderBys()); } String originalQuery = SQLite3Visitor.asString(select); - + generatedQueryString = originalQuery; SQLite3Expression whereClause = gen.generateExpression(); SQLite3UnaryOperation negatedClause = new SQLite3UnaryOperation(UnaryOperator.NOT, whereClause); SQLite3PostfixUnaryOperation notNullClause = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.ISNULL, @@ -122,4 +123,9 @@ private SQLite3Select getSelect(SQLite3Aggregate aggregate, List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setWhereClause(predicate); @@ -37,4 +39,9 @@ public void check() throws SQLException { state); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java index 91977d973..3b39ef4c8 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPGroupByOracle.java @@ -14,6 +14,8 @@ public class SQLite3TLPGroupByOracle extends SQLite3TLPBase { + private String generatedQueryString; + public SQLite3TLPGroupByOracle(SQLite3GlobalState state) { super(state); } @@ -24,7 +26,7 @@ public void check() throws SQLException { select.setGroupByClause(select.getFetchColumns()); select.setWhereClause(null); String originalQueryString = SQLite3Visitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setWhereClause(predicate); @@ -46,4 +48,9 @@ List generateFetchColumns() { .collect(Collectors.toList()); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index 6a960021d..718c0d4ce 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -33,6 +33,7 @@ public class SQLite3TLPHavingOracle implements TestOracle { private final SQLite3GlobalState state; private final ExpectedErrors errors = new ExpectedErrors(); + private String generatedQueryString; public SQLite3TLPHavingOracle(SQLite3GlobalState state) { this.state = state; @@ -61,7 +62,7 @@ public void check() throws SQLException { select.setGroupByClause(groupByColumns); select.setHavingClause(null); String originalQueryString = SQLite3Visitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); SQLite3Expression predicate = gen.getHavingClause(); @@ -84,4 +85,9 @@ public void check() throws SQLException { throw new AssertionError(originalQueryString + ";\n" + combinedString + ";"); } } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java index 95026de86..2f1540f2b 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java @@ -11,6 +11,8 @@ public class SQLite3TLPWhereOracle extends SQLite3TLPBase { + private String generatedQueryString; + public SQLite3TLPWhereOracle(SQLite3GlobalState state) { super(state); } @@ -20,7 +22,7 @@ public void check() throws SQLException { super.check(); select.setWhereClause(null); String originalQueryString = SQLite3Visitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean orderBy = Randomly.getBooleanWithSmallProbability(); @@ -40,4 +42,9 @@ public void check() throws SQLException { state); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } diff --git a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java index 68b4a56c1..23b3cb035 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java @@ -14,6 +14,8 @@ public class TiDBTLPHavingOracle extends TiDBTLPBase implements TestOracle { + private String generatedQueryString; + public TiDBTLPHavingOracle(TiDBGlobalState state) { super(state); TiDBErrors.addExpressionHavingErrors(errors); @@ -32,6 +34,7 @@ public void check() throws SQLException { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); String originalQueryString = TiDBVisitor.asString(select); + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); select.setHavingClause(predicate); @@ -51,4 +54,9 @@ public void check() throws SQLException { protected TiDBExpression generatePredicate() { return gen.generateHavingClause(); } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } } diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java index 4c8496e94..e03b347f3 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java @@ -12,6 +12,8 @@ public class TiDBTLPWhereOracle extends TiDBTLPBase { + private String generatedQueryString; + public TiDBTLPWhereOracle(TiDBGlobalState state) { super(state); TiDBErrors.addExpressionErrors(errors); @@ -22,7 +24,7 @@ public void check() throws SQLException { super.check(); select.setWhereClause(null); String originalQueryString = TiDBVisitor.asString(select); - + generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); @@ -42,4 +44,9 @@ public void check() throws SQLException { state); } + @Override + public String getLastQueryString() { + return generatedQueryString; + } + } From ca70c6a9f9346b7eab8ecc44573e890068afd47b Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 18:00:14 +0800 Subject: [PATCH 267/856] fix: broken grammar of LIKELIHOOD (#650) --- src/sqlancer/sqlite3/SQLite3Errors.java | 1 - .../sqlite3/gen/SQLite3ExpressionGenerator.java | 11 +++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index b872fcd1d..6cd4c40c9 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -21,7 +21,6 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("FTS expression tree is too large"); errors.add("String or BLOB exceeds size limit"); errors.add("[SQLITE_ERROR] SQL error or missing database (integer overflow)"); - errors.add("second argument to likelihood() must be a constant between 0.0 and 1.0"); errors.add("ORDER BY term out of range"); errors.add("GROUP BY term out of range"); errors.add("not authorized"); // load_extension diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index d21f00af3..5e7e32cb6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -547,6 +547,12 @@ private SQLite3Expression getFunction(SQLite3GlobalState globalState, int depth) nrArgs += Randomly.smallNumber(); } List expressions = randomFunction.generateArguments(nrArgs, depth + 1, this); + // The second argument of LIKELIHOOD must be a float number within 0.0 -1.0 + if (randomFunction == AnyFunction.LIKELIHOOD) { + SQLite3Expression lastArg = SQLite3Constant.createRealConstant(Randomly.getPercentage()); + expressions.remove(expressions.size() - 1); + expressions.add(lastArg); + } return new SQLite3Expression.Function(randomFunction.toString(), expressions.toArray(new SQLite3Expression[0])); } @@ -604,6 +610,11 @@ private SQLite3Expression getComputableFunction(int depth) { args[i] = new SQLite3Distinct(args[i]); } } + // The second argument of LIKELIHOOD must be a float number within 0.0 -1.0 + if (func == ComputableFunction.LIKELIHOOD) { + SQLite3Expression lastArg = SQLite3Constant.createRealConstant(Randomly.getPercentage()); + args[args.length - 1] = lastArg; + } return new SQLite3Function(func, args); } From 977edb45d11410286753dd3102fff34263239203 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 17 Dec 2022 19:09:47 +0800 Subject: [PATCH 268/856] tidb: turn off bug switch (#638) * tidb: turn off bug switch Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang * improve code Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBBugs.java | 43 ------------------- .../tidb/TiDBExpressionGenerator.java | 9 ---- .../tidb/gen/TiDBAlterTableGenerator.java | 7 --- src/sqlancer/tidb/gen/TiDBHintGenerator.java | 4 -- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 12 ++---- .../tidb/visitor/TiDBToStringVisitor.java | 5 --- 6 files changed, 3 insertions(+), 77 deletions(-) diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index a4f5cc2dd..d56629e44 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -2,49 +2,6 @@ // do not make the fields final to avoid warnings public final class TiDBBugs { - - // https://github.com/pingcap/tidb/issues/15987 - public static boolean bug15987; - - // // https://github.com/pingcap/tidb/issues/15988 - public static boolean bug15988; - - // https://github.com/pingcap/tidb/issues/16028 - public static boolean bug16028; - - // https://github.com/pingcap/tidb/issues/16020 - public static boolean bug16020; - - // https://github.com/pingcap/tidb/issues/15990 - public static boolean bug15990; - - // https://github.com/pingcap/tidb/issues/15844 - public static boolean bug15844; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/10 - public static boolean bug10; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/14 - public static boolean bug14; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/15 - public static boolean bug15; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/16 - public static boolean bug16; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/19 - public static boolean bug19; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/48 - public static boolean bug48; - - // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/50 - public static boolean bug50; - - // https://github.com/citusdata/citus/issues/4079 - public static boolean bug4079; - // https://github.com/pingcap/tidb/issues/35677 public static boolean bug35677 = true; diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index 535647c53..a8fb75da0 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -63,9 +63,6 @@ protected TiDBExpression generateExpression(int depth) { } switch (Randomly.fromOptions(Gen.values())) { case DEFAULT: - if (TiDBBugs.bug15) { - throw new IgnoreMeException(); - } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } @@ -92,9 +89,6 @@ protected TiDBExpression generateExpression(int depth) { return new TiDBBinaryBitOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryBitOperator.getRandom()); case BINARY_LOGICAL: - if (TiDBBugs.bug48) { - throw new IgnoreMeException(); - } return new TiDBBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), TiDBBinaryLogicalOperator.getRandom()); case CAST: @@ -102,9 +96,6 @@ protected TiDBExpression generateExpression(int depth) { "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 "DECIMAL", "SIGNED", "UNSIGNED" /* https://github.com/pingcap/tidb/issues/16028 */)); case CASE: - if (TiDBBugs.bug19) { - throw new IgnoreMeException(); - } int nr = Randomly.fromOptions(1, 2); return new TiDBCase(generateExpression(depth + 1), generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), generateExpression(depth + 1)); diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index a327dde39..b6f65a780 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; @@ -39,9 +38,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append(" "); switch (a) { case MODIFY_COLUMN: - if (TiDBBugs.bug10) { - throw new IgnoreMeException(); - } sb.append("MODIFY "); sb.append(column.getName()); sb.append(" "); @@ -92,9 +88,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("'Defining a virtual generated column as primary key' is not supported for generated columns"); break; case CHANGE: - if (TiDBBugs.bug10) { - throw new IgnoreMeException(); - } sb.append(" CHANGE "); sb.append(column.getName()); sb.append(" "); diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index 355cb2934..06105ed4c 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -6,7 +6,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.schema.TableIndex; -import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.ast.TiDBSelect; import sqlancer.tidb.ast.TiDBText; @@ -61,9 +60,6 @@ private void generate() { tablesHint("INL_JOIN"); break; case INL_HASH_JOIN: - if (TiDBBugs.bug50) { - throw new IgnoreMeException(); - } tablesHint("INL_HASH_JOIN"); break; case INL_MERGE_JOIN: diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 25d2cbb99..4ca210b90 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -8,7 +8,6 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; @@ -113,8 +112,7 @@ && canUseAsUnique(type) && !isGeneratedColumn) { errors.add(" used in key specification without a key length"); } sb.append(")"); - if (Randomly.getBooleanWithRatherLowProbability() - && !TiDBBugs.bug14 /* there are also a number of unresolved other partitioning bugs */) { + if (Randomly.getBooleanWithRatherLowProbability()) { sb.append("PARTITION BY HASH("); sb.append(TiDBVisitor.asString(gen.generateExpression())); sb.append(") "); @@ -127,9 +125,6 @@ && canUseAsUnique(type) && !isGeneratedColumn) { errors.add("A UNIQUE INDEX must include all columns in the table's partitioning function"); errors.add("is of a not allowed type for this type of partitioning"); errors.add("The PARTITION function returns the wrong type"); - if (TiDBBugs.bug16) { - errors.add("UnknownType: *ast.WhenClause"); - } } } @@ -144,11 +139,10 @@ private void appendType(StringBuilder sb, TiDBCompositeDataType type) { } private void appendSizeSpecifiers(StringBuilder sb, TiDBDataType type) { - if (type.isNumeric() && Randomly.getBoolean() && !TiDBBugs.bug16028) { + if (type.isNumeric() && Randomly.getBoolean()) { sb.append(" UNSIGNED"); } - if (type.isNumeric() && Randomly.getBoolean() - && !TiDBBugs.bug16028 /* seems to be the same bug as https://github.com/pingcap/tidb/issues/16028 */) { + if (type.isNumeric() && Randomly.getBoolean()) { sb.append(" ZEROFILL"); } } diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index 5e0c56841..8bd47ed70 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -1,9 +1,7 @@ package sqlancer.tidb.visitor; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; -import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.ast.TiDBAggregate; import sqlancer.tidb.ast.TiDBCase; import sqlancer.tidb.ast.TiDBCastOperation; @@ -130,9 +128,6 @@ public void visit(TiDBJoin join) { sb.append("LEFT "); break; case RIGHT: - if (TiDBBugs.bug15844) { - throw new IgnoreMeException(); - } sb.append("RIGHT "); break; default: From ec1cbe243eba8470706fe09b6c4d785c96be4f76 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 20:09:32 +0800 Subject: [PATCH 269/856] fix: broken grammar of aggreate in SQLite (#651) --- .../sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 90ca0d96b..849a8640f 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -66,7 +66,9 @@ public void check() throws SQLException { SQLite3Select leftSelect = getSelect(aggregate, from, whereClause); SQLite3Select middleSelect = getSelect(aggregate, from, negatedClause); SQLite3Select rightSelect = getSelect(aggregate, from, notNullClause); - String metamorphicText = "SELECT " + aggregate.getFunc().toString() + "(aggr) FROM ("; + String aggreateMethod = aggregate.getFunc() == SQLite3AggregateFunction.COUNT_ALL + ? SQLite3AggregateFunction.COUNT.toString() : aggregate.getFunc().toString(); + String metamorphicText = "SELECT " + aggreateMethod + "(aggr) FROM ("; metamorphicText += SQLite3Visitor.asString(leftSelect) + " UNION ALL " + SQLite3Visitor.asString(middleSelect) + " UNION ALL " + SQLite3Visitor.asString(rightSelect); metamorphicText += ")"; From 5533c4e3c32a7f474bf1e047bda5f529fb9ef7ba Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 20:38:37 +0800 Subject: [PATCH 270/856] Bump SQLite version to 3.40.0.0 (#652) * bump SQLite to 3.40.0.0 * Support RIGHT/FULL JOINs --- pom.xml | 2 +- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 6 ++++++ src/sqlancer/sqlite3/ast/SQLite3Expression.java | 2 +- .../sqlite3/gen/SQLite3ExpressionGenerator.java | 14 ++++++++++---- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pom.xml b/pom.xml index 1ddb71c02..7b163a49f 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ org.xerial sqlite-jdbc - 3.36.0.3 + 3.40.0.0 mysql diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index f81259732..475c16335 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -237,6 +237,12 @@ public void visit(Join join) { case OUTER: sb.append("LEFT OUTER"); break; + case RIGHT: + sb.append("RIGHT OUTER"); + break; + case FULL: + sb.append("FULL OUTER"); + break; default: throw new AssertionError(join.getType()); } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 748c70375..8f99da8e5 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -131,7 +131,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { public static class Join extends SQLite3Expression { public enum JoinType { - INNER, CROSS, OUTER, NATURAL; + INNER, CROSS, OUTER, NATURAL, RIGHT, FULL; } private final SQLite3Table table; diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 5e7e32cb6..89e7e5860 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -139,19 +139,25 @@ public List getRandomJoinClauses(List tables) { if (!globalState.getDbmsSpecificOptions().testJoins) { return joinStatements; } + List options = new ArrayList<>(Arrays.asList(JoinType.values())); if (Randomly.getBoolean() && tables.size() > 1) { int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1) { + options.remove(JoinType.NATURAL); + } for (int i = 0; i < nrJoinClauses; i++) { SQLite3Expression joinClause = generateExpression(); SQLite3Table table = Randomly.fromList(tables); tables.remove(table); - JoinType options; - options = Randomly.fromOptions(JoinType.INNER, JoinType.CROSS, JoinType.OUTER, JoinType.NATURAL); - if (options == JoinType.NATURAL) { + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { // NATURAL joins do not have an ON clause joinClause = null; } - Join j = new SQLite3Expression.Join(table, joinClause, options); + Join j = new SQLite3Expression.Join(table, joinClause, selectedOption); joinStatements.add(j); } From a535520766808fd5bb71db41a0351e9d1f0d5331 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Dec 2022 21:22:32 +0800 Subject: [PATCH 271/856] fix: temp table generation (#653) --- src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index ffa3af6cd..5d8a40547 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -31,7 +31,7 @@ public static SQLQueryAdapter dropView(SQLite3GlobalState globalState) { public static SQLQueryAdapter generate(SQLite3GlobalState globalState) throws SQLException { StringBuilder sb = new StringBuilder(); sb.append("CREATE"); - if (Randomly.getBoolean()) { + if (globalState.getDbmsSpecificOptions().testTempTables && Randomly.getBoolean()) { sb.append(" "); sb.append(Randomly.fromOptions("TEMP", "TEMPORARY")); } From a195cdd0b4a97cf1113c4bded2e1a41281d4a26a Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 13:20:47 +0800 Subject: [PATCH 272/856] opt: support new features in SQLite (#654) * opt: support maximum number of table/index * opt: support explain interface --- src/sqlancer/sqlite3/SQLite3Options.java | 7 +++ src/sqlancer/sqlite3/SQLite3Provider.java | 60 +++++++++++-------- ...Lite3CreateVirtualRtreeTabelGenerator.java | 9 +++ .../sqlite3/gen/SQLite3ExplainGenerator.java | 7 +++ ...SQLite3CreateVirtualFTSTableGenerator.java | 10 ++++ .../gen/ddl/SQLite3IndexGenerator.java | 4 ++ .../gen/ddl/SQLite3TableGenerator.java | 9 +++ .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 5 ++ 8 files changed, 85 insertions(+), 26 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 6df9c378c..e740a5eea 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -90,6 +90,13 @@ public class SQLite3Options implements DBMSSpecificOptions "--execute-queries" }, description = "Specifies whether the query in the fuzzer should be executed", arity = 1) public boolean executeQuery = true; + @Parameter(names = { + "--max-num-tables" }, description = "The maximum number of tables/virtual tables/ rtree tables/ views that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + public enum SQLite3OracleFactory implements OracleFactory { PQS { @Override diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index eb88a0e0a..1cb5e0633 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -58,47 +58,50 @@ public SQLite3Provider() { } public enum Action implements AbstractAction { - PRAGMA(SQLite3PragmaGenerator::insertPragma), // - INDEX(SQLite3IndexGenerator::insertIndex), // - INSERT(SQLite3InsertGenerator::insertRow), // - VACUUM(SQLite3VacuumGenerator::executeVacuum), // - REINDEX(SQLite3ReindexGenerator::executeReindex), // - ANALYZE(SQLite3AnalyzeGenerator::generateAnalyze), // - DELETE(SQLite3DeleteGenerator::deleteContent), // + PRAGMA(SQLite3PragmaGenerator::insertPragma), // 0 + CREATE_INDEX(SQLite3IndexGenerator::insertIndex), // 1 + CREATE_VIEW(SQLite3ViewGenerator::generate), // 2 + CREATE_TRIGGER(SQLite3CreateTriggerGenerator::create), // 3 + CREATE_TABLE(SQLite3TableGenerator::createRandomTableStatement), // 4 + CREATE_VIRTUALTABLE(SQLite3CreateVirtualFTSTableGenerator::createRandomTableStatement), // 5 + CREATE_RTREETABLE(SQLite3CreateVirtualRtreeTabelGenerator::createRandomTableStatement), // 6 + INSERT(SQLite3InsertGenerator::insertRow), // 7 + DELETE(SQLite3DeleteGenerator::deleteContent), // 8 + ALTER(SQLite3AlterTable::alterTable), // 9 + UPDATE(SQLite3UpdateGenerator::updateRow), // 10 + DROP_INDEX(SQLite3DropIndexGenerator::dropIndex), // 11 + DROP_TABLE(SQLite3DropTableGenerator::dropTable), // 12 + DROP_VIEW(SQLite3ViewGenerator::dropView), // 13 + VACUUM(SQLite3VacuumGenerator::executeVacuum), // 14 + REINDEX(SQLite3ReindexGenerator::executeReindex), // 15 + ANALYZE(SQLite3AnalyzeGenerator::generateAnalyze), // 16 + EXPLAIN(SQLite3ExplainGenerator::explain), // 17 + CHECK_RTREE_TABLE((g) -> { + SQLite3Table table = g.getSchema().getRandomTableOrBailout(t -> t.getName().startsWith("r")); + String format = String.format("SELECT rtreecheck('%s');", table.getName()); + return new SQLQueryAdapter(format, ExpectedErrors.from("The database file is locked")); + }), // 18 + VIRTUAL_TABLE_ACTION(SQLite3VirtualFTSTableCommandGenerator::create), // 19 + MANIPULATE_STAT_TABLE(SQLite3StatTableGenerator::getQuery), // 20 TRANSACTION_START(SQLite3TransactionGenerator::generateBeginTransaction) { @Override public boolean canBeRetried() { return false; } - }, // - ALTER(SQLite3AlterTable::alterTable), // - DROP_INDEX(SQLite3DropIndexGenerator::dropIndex), // - UPDATE(SQLite3UpdateGenerator::updateRow), // + }, // 21 ROLLBACK_TRANSACTION(SQLite3TransactionGenerator::generateRollbackTransaction) { @Override public boolean canBeRetried() { return false; } - }, // + }, // 22 COMMIT(SQLite3TransactionGenerator::generateCommit) { @Override public boolean canBeRetried() { return false; } - }, // - DROP_TABLE(SQLite3DropTableGenerator::dropTable), // - DROP_VIEW(SQLite3ViewGenerator::dropView), // - EXPLAIN(SQLite3ExplainGenerator::explain), // - CHECK_RTREE_TABLE((g) -> { - SQLite3Table table = g.getSchema().getRandomTableOrBailout(t -> t.getName().startsWith("r")); - String format = String.format("SELECT rtreecheck('%s');", table.getName()); - return new SQLQueryAdapter(format, ExpectedErrors.from("The database file is locked")); - }), // - VIRTUAL_TABLE_ACTION(SQLite3VirtualFTSTableCommandGenerator::create), // - CREATE_VIEW(SQLite3ViewGenerator::generate), // - CREATE_TRIGGER(SQLite3CreateTriggerGenerator::create), // - MANIPULATE_STAT_TABLE(SQLite3StatTableGenerator::getQuery); + }; // 23 private final SQLQueryProvider sqlQueryProvider; @@ -146,7 +149,7 @@ private static int mapActions(SQLite3GlobalState globalState, Action a) { case MANIPULATE_STAT_TABLE: nrPerformed = r.getInteger(0, 5); break; - case INDEX: + case CREATE_INDEX: nrPerformed = r.getInteger(0, 5); break; case VIRTUAL_TABLE_ACTION: @@ -156,6 +159,11 @@ private static int mapActions(SQLite3GlobalState globalState, Action a) { case PRAGMA: nrPerformed = r.getInteger(0, 20); break; + case CREATE_TABLE: + case CREATE_VIRTUALTABLE: + case CREATE_RTREETABLE: + nrPerformed = 0; + break; case TRANSACTION_START: case REINDEX: case ANALYZE: diff --git a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java index 08effcdc9..61f821382 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3CreateVirtualRtreeTabelGenerator.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -15,6 +16,14 @@ public final class SQLite3CreateVirtualRtreeTabelGenerator { private SQLite3CreateVirtualRtreeTabelGenerator() { } + public static SQLQueryAdapter createRandomTableStatement(SQLite3GlobalState globalState) { + if (globalState.getSchema().getTables().getTables() + .size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return createTableStatement(globalState.getSchema().getFreeRtreeTableName(), globalState); + } + public static SQLQueryAdapter createTableStatement(String rTreeTableName, SQLite3GlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); List columns = new ArrayList<>(); diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java index e9ede64d9..8c00b3231 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExplainGenerator.java @@ -26,4 +26,11 @@ public static SQLQueryAdapter explain(SQLite3GlobalState globalState) throws Exc return new SQLQueryAdapter(sb.toString(), query.getExpectedErrors()); } + public static String explain(String selectStr) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN QUERY PLAN "); + sb.append(selectStr); + return sb.toString(); + } + } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java index a0f79fa7e..3ae9d16d5 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3CreateVirtualFTSTableGenerator.java @@ -4,10 +4,12 @@ import java.util.Arrays; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; public class SQLite3CreateVirtualFTSTableGenerator { @@ -20,6 +22,14 @@ public SQLite3CreateVirtualFTSTableGenerator(String tableName, Randomly r) { this.r = r; } + public static SQLQueryAdapter createRandomTableStatement(SQLite3GlobalState globalState) { + if (globalState.getSchema().getTables().getTables() + .size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return createTableStatement(globalState.getSchema().getFreeVirtualTableName(), globalState.getRandomly()); + } + public static SQLQueryAdapter createTableStatement(String tableName, Randomly r) { return new SQLite3CreateVirtualFTSTableGenerator(tableName, r).create(); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java index 0a33016fd..c81d895ac 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3IndexGenerator.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -23,6 +24,9 @@ public class SQLite3IndexGenerator { private final SQLite3GlobalState globalState; public static SQLQueryAdapter insertIndex(SQLite3GlobalState globalState) throws SQLException { + if (globalState.getSchema().getIndexNames().size() >= globalState.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } return new SQLite3IndexGenerator(globalState).create(); } diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index cc4731911..6c27a5d58 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -46,6 +47,14 @@ public SQLite3TableGenerator(String tableName, SQLite3GlobalState globalState) { this.existingSchema = globalState.getSchema(); } + public static SQLQueryAdapter createRandomTableStatement(SQLite3GlobalState globalState) { + if (globalState.getSchema().getTables().getTables() + .size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return createTableStatement(globalState.getSchema().getFreeTableName(), globalState); + } + public static SQLQueryAdapter createTableStatement(String tableName, SQLite3GlobalState globalState) { SQLite3TableGenerator sqLite3TableGenerator = new SQLite3TableGenerator(tableName, globalState); sqLite3TableGenerator.start(); diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 5d8a40547..1c483e59d 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -2,6 +2,7 @@ import java.sql.SQLException; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -29,6 +30,10 @@ public static SQLQueryAdapter dropView(SQLite3GlobalState globalState) { } public static SQLQueryAdapter generate(SQLite3GlobalState globalState) throws SQLException { + if (globalState.getSchema().getTables().getTables() + .size() >= globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } StringBuilder sb = new StringBuilder(); sb.append("CREATE"); if (globalState.getDbmsSpecificOptions().testTempTables && Randomly.getBoolean()) { From 7a083b6a9d8774c75ffea8d8be31fb10e4b7d9eb Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 14:51:25 +0800 Subject: [PATCH 273/856] fix: null pointer reference (#656) --- .../cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index 3cf614cea..0dae9331d 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -85,8 +85,8 @@ public void check() throws SQLException { state.getState().getLocalState().log( "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); - if (firstResult == null && secondResult != null - || firstResult != null && (!firstResult.contentEquals(secondResult) + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && secondResult != null && (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { if (secondResult.contains("Inf")) { throw new IgnoreMeException(); // FIXME: average computation From eb2ec91e04be5af671486e25ea54abdce4f79ae2 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 15:49:57 +0800 Subject: [PATCH 274/856] opt: some new features in CockroachDB (#657) * opt: maximum number of tables/indexes in CockroachDB * opt: support drop table/view * opt: update Actions --- .../cockroachdb/CockroachDBOptions.java | 6 +++ .../cockroachdb/CockroachDBProvider.java | 21 ++++++---- .../gen/CockroachDBDropTableGenerator.java | 35 ++++++++++++++++ .../gen/CockroachDBDropViewGenerator.java | 41 +++++++++++++++++++ .../gen/CockroachDBIndexGenerator.java | 4 ++ .../gen/CockroachDBTableGenerator.java | 3 ++ 6 files changed, 102 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java create mode 100644 src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 38b845781..4fdf52b0e 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -103,6 +103,12 @@ public TestOracle create(CockroachDBGlobalState globalSt "--increased-vectorization" }, description = "Generate VECTORIZE=on with a higher probability (which found a number of bugs in the past)") public boolean makeVectorizationMoreLikely = true; + @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + @Override public List getTestOracleFactory() { return Arrays.asList(oracle); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index abc7e6e20..64da45bb2 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -6,7 +6,6 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -23,6 +22,8 @@ import sqlancer.cockroachdb.gen.CockroachDBCommentOnGenerator; import sqlancer.cockroachdb.gen.CockroachDBCreateStatisticsGenerator; import sqlancer.cockroachdb.gen.CockroachDBDeleteGenerator; +import sqlancer.cockroachdb.gen.CockroachDBDropTableGenerator; +import sqlancer.cockroachdb.gen.CockroachDBDropViewGenerator; import sqlancer.cockroachdb.gen.CockroachDBIndexGenerator; import sqlancer.cockroachdb.gen.CockroachDBInsertGenerator; import sqlancer.cockroachdb.gen.CockroachDBRandomQuerySynthesizer; @@ -45,15 +46,17 @@ public CockroachDBProvider() { } public enum Action { - INSERT(CockroachDBInsertGenerator::insert), // - TRUNCATE(CockroachDBTruncateGenerator::truncate), // + CREATE_TABLE(CockroachDBTableGenerator::generate), CREATE_INDEX(CockroachDBIndexGenerator::create), // + CREATE_VIEW(CockroachDBViewGenerator::generate), // CREATE_STATISTICS(CockroachDBCreateStatisticsGenerator::create), // - SET_SESSION(CockroachDBSetSessionGenerator::create), // - CREATE_INDEX(CockroachDBIndexGenerator::create), // + INSERT(CockroachDBInsertGenerator::insert), // UPDATE(CockroachDBUpdateGenerator::gen), // - CREATE_VIEW(CockroachDBViewGenerator::generate), // + SET_SESSION(CockroachDBSetSessionGenerator::create), // SET_CLUSTER_SETTING(CockroachDBSetClusterSettingGenerator::create), // DELETE(CockroachDBDeleteGenerator::delete), // + TRUNCATE(CockroachDBTruncateGenerator::truncate), // + DROP_TABLE(CockroachDBDropTableGenerator::drop), // + DROP_VIEW(CockroachDBDropViewGenerator::drop), // COMMENT_ON(CockroachDBCommentOnGenerator::comment), // SHOW(CockroachDBShowGenerator::show), // TRANSACTION((g) -> { @@ -65,8 +68,7 @@ public enum Action { ExpectedErrors errors = new ExpectedErrors(); if (Randomly.getBoolean()) { sb.append("("); - sb.append(Randomly.nonEmptySubset("VERBOSE", "TYPES", "OPT", "DISTSQL", "VEC").stream() - .collect(Collectors.joining(", "))); + sb.append(Randomly.fromOptions("VERBOSE", "TYPES", "OPT", "DISTSQL", "VEC")); sb.append(") "); errors.add("cannot set EXPLAIN mode more than once"); errors.add("unable to vectorize execution plan"); @@ -199,6 +201,9 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio */ break; case TRANSACTION: + case CREATE_TABLE: + case DROP_TABLE: + case DROP_VIEW: nrPerformed = 0; // r.getInteger(0, 0); break; default: diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java new file mode 100644 index 000000000..3834c8757 --- /dev/null +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDropTableGenerator.java @@ -0,0 +1,35 @@ +package sqlancer.cockroachdb.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public final class CockroachDBDropTableGenerator { + + private CockroachDBDropTableGenerator() { + } + + public static SQLQueryAdapter drop(CockroachDBGlobalState globalState) { + if (globalState.getSchema().getTables(t -> !t.isView()).size() <= 1) { + throw new IgnoreMeException(); + } + + ExpectedErrors errors = new ExpectedErrors(); + errors.add("is referenced by foreign key"); + + StringBuilder sb = new StringBuilder(); + sb.append("DROP"); + sb.append(" TABLE"); + sb.append(" "); + sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), true); + } + +} diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java new file mode 100644 index 000000000..5d440b564 --- /dev/null +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDropViewGenerator.java @@ -0,0 +1,41 @@ +package sqlancer.cockroachdb.gen; + +import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; + +public final class CockroachDBDropViewGenerator { + + private CockroachDBDropViewGenerator() { + } + + public static SQLQueryAdapter drop(CockroachDBGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + errors.add("is referenced by foreign key"); + + StringBuilder sb = new StringBuilder(); + sb.append("DROP"); + if (Randomly.getBoolean()) { + sb.append(" MATERIALIZED"); + } + sb.append(" VIEW"); + sb.append(" "); + if (Randomly.getBooleanWithRatherLowProbability()) { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(globalState.getSchema().getRandomTable(t -> t.isView()).getName()); + } + } else { + sb.append(globalState.getSchema().getRandomTable(t -> t.isView()).getName()); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), true); + } + +} diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index e5b708338..4db3588ac 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -2,6 +2,7 @@ import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -17,6 +18,9 @@ public CockroachDBIndexGenerator(CockroachDBGlobalState globalState) { } public static SQLQueryAdapter create(CockroachDBGlobalState s) { + if (s.getSchema().getIndexCount() >= s.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } return new CockroachDBIndexGenerator(s).getQuery(); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index 4e99ab31a..c32fa76eb 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -29,6 +29,9 @@ public CockroachDBTableGenerator(CockroachDBGlobalState globalState) { } public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } return new CockroachDBTableGenerator(globalState).getQuery(); } From 1ca72afc1bc76f0584c3db84a0b1ef7a55a41e7e Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 16:13:14 +0800 Subject: [PATCH 275/856] fix: connection string cockroachdb (#658) --- src/sqlancer/cockroachdb/CockroachDBProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 64da45bb2..580fa1ebf 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -278,7 +278,7 @@ public SQLConnection createDatabase(CockroachDBGlobalState globalState) throws S s.execute(createDatabaseCommand); } con.close(); - con = DriverManager.getConnection("jdbc:postgresql://localhost:26257/" + databaseName, + con = DriverManager.getConnection(String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); return new SQLConnection(con); } From 00cda2ec1538f8b1b2af8bf3770d233331447a48 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Dec 2022 16:46:19 +0800 Subject: [PATCH 276/856] fix: switch off vectorization cockroachdb (#659) --- src/sqlancer/cockroachdb/CockroachDBOptions.java | 4 ---- src/sqlancer/cockroachdb/CockroachDBProvider.java | 3 --- 2 files changed, 7 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index 4fdf52b0e..c41b018ee 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -99,10 +99,6 @@ public TestOracle create(CockroachDBGlobalState globalSt @Parameter(names = { "--test-temp-tables" }, description = "Test TEMPORARY tables") public boolean testTempTables; // default: false https://github.com/cockroachdb/cockroach/issues/85388 - @Parameter(names = { - "--increased-vectorization" }, description = "Generate VECTORIZE=on with a higher probability (which found a number of bugs in the past)") - public boolean makeVectorizationMoreLikely = true; - @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables that can be created") public int maxNumTables = 10; diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 580fa1ebf..a79d578db 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -247,9 +247,6 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio } total--; } - if (globalState.getDbmsSpecificOptions().makeVectorizationMoreLikely && Randomly.getBoolean()) { - manager.execute(new SQLQueryAdapter("SET vectorize=on;")); - } } @Override From 0168bb1d8a7a5d1ab68f06248d549c09c59a8203 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Sun, 18 Dec 2022 17:07:27 +0800 Subject: [PATCH 277/856] opt: maximum number of tables/indexes tidb --- src/sqlancer/tidb/TiDBOptions.java | 7 ++++++- src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 4 ++++ src/sqlancer/tidb/gen/TiDBTableGenerator.java | 8 ++++++++ src/sqlancer/tidb/gen/TiDBViewGenerator.java | 3 +++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 51afa2a9a..764c4bfd9 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -23,6 +23,12 @@ public class TiDBOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 4000; + @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables/views that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + @Parameter(names = "--oracle") public List oracle = Arrays.asList(TiDBOracleFactory.QUERY_PARTITIONING); @@ -55,5 +61,4 @@ public TestOracle create(TiDBGlobalState globalState) throws SQ public List getTestOracleFactory() { return oracle; } - } diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index 3444d3281..cb870a2df 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -16,6 +17,9 @@ private TiDBIndexGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { + if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } ExpectedErrors errors = new ExpectedErrors(); TiDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 4ca210b90..0ce8190e7 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -23,6 +24,13 @@ public class TiDBTableGenerator { private boolean primaryKeyAsTableConstraints; private final ExpectedErrors errors = new ExpectedErrors(); + public static SQLQueryAdapter createRandomTableStatement(TiDBGlobalState globalState) throws SQLException { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return new TiDBTableGenerator().getQuery(globalState); + } + public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { errors.add("Information schema is changed during the execution of the statement"); String tableName = globalState.getSchema().getFreeTableName(); diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index 9da8e433e..fdbbc0839 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -13,6 +13,9 @@ private TiDBViewGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } int nrColumns = Randomly.smallNumber() + 1; StringBuilder sb = new StringBuilder("CREATE "); if (Randomly.getBoolean()) { From 60394c128f39d8eae8e0a6478d1f3a3d30963d98 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Sun, 18 Dec 2022 17:07:56 +0800 Subject: [PATCH 278/856] opt: support dropping tables/views tidb --- .../tidb/gen/TiDBDropTableGenerator.java | 25 +++++++++++++++++++ .../tidb/gen/TiDBDropViewGenerator.java | 25 +++++++++++++++++++ 2 files changed, 50 insertions(+) create mode 100644 src/sqlancer/tidb/gen/TiDBDropTableGenerator.java create mode 100644 src/sqlancer/tidb/gen/TiDBDropViewGenerator.java diff --git a/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java b/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java new file mode 100644 index 000000000..727fffffc --- /dev/null +++ b/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.tidb.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; + +public final class TiDBDropTableGenerator { + + private TiDBDropTableGenerator() { + } + + public static SQLQueryAdapter dropTable(TiDBGlobalState globalState) { + if (globalState.getSchema().getTables(t -> !t.isView()).size() <= 1) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> !t.isView()).getName()); + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java new file mode 100644 index 000000000..48bfc8a46 --- /dev/null +++ b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.tidb.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; + +public final class TiDBDropViewGenerator { + + private TiDBDropViewGenerator() { + } + + public static SQLQueryAdapter dropView(TiDBGlobalState globalState) { + if (globalState.getSchema().getTables(t -> t.isView()).size() == 0) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP VIEW "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> t.isView()).getName()); + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} From fb93140c112281b72cc7a3e2db9df4eeae740879 Mon Sep 17 00:00:00 2001 From: bajinsheng <43855029@qq.com> Date: Sun, 18 Dec 2022 17:08:25 +0800 Subject: [PATCH 279/856] opt: action update tidb --- src/sqlancer/tidb/TiDBProvider.java | 41 +++++++++++++---------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index af41fb143..dd02a474f 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -16,16 +16,16 @@ import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; import sqlancer.tidb.gen.TiDBDeleteGenerator; +import sqlancer.tidb.gen.TiDBDropTableGenerator; +import sqlancer.tidb.gen.TiDBDropViewGenerator; import sqlancer.tidb.gen.TiDBIndexGenerator; import sqlancer.tidb.gen.TiDBInsertGenerator; -import sqlancer.tidb.gen.TiDBRandomQuerySynthesizer; import sqlancer.tidb.gen.TiDBSetGenerator; import sqlancer.tidb.gen.TiDBTableGenerator; import sqlancer.tidb.gen.TiDBUpdateGenerator; @@ -39,25 +39,20 @@ public TiDBProvider() { } public enum Action implements AbstractAction { - INSERT(TiDBInsertGenerator::getQuery), // - ANALYZE_TABLE(TiDBAnalyzeTableGenerator::getQuery), // - TRUNCATE((g) -> new SQLQueryAdapter("TRUNCATE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), // - CREATE_INDEX(TiDBIndexGenerator::getQuery), // - DELETE(TiDBDeleteGenerator::getQuery), // - SET(TiDBSetGenerator::getQuery), // - UPDATE(TiDBUpdateGenerator::getQuery), // + CREATE_TABLE(TiDBTableGenerator::createRandomTableStatement), // 0 + CREATE_INDEX(TiDBIndexGenerator::getQuery), // 1 + VIEW_GENERATOR(TiDBViewGenerator::getQuery), // 2 + INSERT(TiDBInsertGenerator::getQuery), // 3 + ALTER_TABLE(TiDBAlterTableGenerator::getQuery), // 4 + TRUNCATE((g) -> new SQLQueryAdapter("TRUNCATE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), // 5 + UPDATE(TiDBUpdateGenerator::getQuery), // 6 + DELETE(TiDBDeleteGenerator::getQuery), // 7 + SET(TiDBSetGenerator::getQuery), // 8 ADMIN_CHECKSUM_TABLE( - (g) -> new SQLQueryAdapter("ADMIN CHECKSUM TABLE " + g.getSchema().getRandomTable().getName())), // - VIEW_GENERATOR(TiDBViewGenerator::getQuery), // - ALTER_TABLE(TiDBAlterTableGenerator::getQuery), // - EXPLAIN((g) -> { - ExpectedErrors errors = new ExpectedErrors(); - TiDBErrors.addExpressionErrors(errors); - TiDBErrors.addExpressionHavingErrors(errors); - return new SQLQueryAdapter( - "EXPLAIN " + TiDBRandomQuerySynthesizer.generate(g, Randomly.smallNumber() + 1).getQueryString(), - errors); - }); + (g) -> new SQLQueryAdapter("ADMIN CHECKSUM TABLE " + g.getSchema().getRandomTable().getName())), // 9 + ANALYZE_TABLE(TiDBAnalyzeTableGenerator::getQuery), // 10 + DROP_TABLE(TiDBDropTableGenerator::dropTable), // 11 + DROP_VIEW(TiDBDropViewGenerator::dropView); // 12 private final SQLQueryProvider sqlQueryProvider; @@ -87,8 +82,6 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { case CREATE_INDEX: return r.getInteger(0, 2); case INSERT: - case EXPLAIN: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case TRUNCATE: case DELETE: case ADMIN_CHECKSUM_TABLE: @@ -101,6 +94,10 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { return r.getInteger(0, 2); case ALTER_TABLE: return r.getInteger(0, 10); // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/10 + case CREATE_TABLE: + case DROP_TABLE: + case DROP_VIEW: + return 0; default: throw new AssertionError(a); } From 38bf6407a273d7900d29bea0aca9a6404389bf58 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 19 Dec 2022 16:28:32 +0800 Subject: [PATCH 280/856] Query Plan Guidance Framework (#661) * QPG framework * QPG for SQLite * QPG for CockroachDB * QPG for TiDB --- src/sqlancer/DatabaseProvider.java | 12 ++ src/sqlancer/Main.java | 54 +++++- src/sqlancer/MainOptions.java | 35 ++++ src/sqlancer/ProviderAdapter.java | 168 +++++++++++++++++- .../cockroachdb/CockroachDBProvider.java | 55 ++++++ src/sqlancer/common/DBMSCommon.java | 12 ++ src/sqlancer/sqlite3/SQLite3Provider.java | 58 ++++++ src/sqlancer/tidb/TiDBProvider.java | 41 +++++ 8 files changed, 430 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/DatabaseProvider.java b/src/sqlancer/DatabaseProvider.java index 48afb00f5..d169324fa 100644 --- a/src/sqlancer/DatabaseProvider.java +++ b/src/sqlancer/DatabaseProvider.java @@ -32,6 +32,18 @@ public interface DatabaseProvider, O extends DBMS */ Reproducer generateAndTestDatabase(G globalState) throws Exception; + /** + * The experimental feature: Query Plan Guidance. + * + * @param globalState + * the state created and is valid for this method call. + * + * @throws Exception + * if testing fails. + * + */ + void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws Exception; + C createDatabase(G globalState) throws Exception; /** diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index dfb2bb89b..29610f266 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -51,10 +51,13 @@ public static final class StateLogger { private final File loggerFile; private File curFile; + private File queryPlanFile; private FileWriter logFileWriter; public FileWriter currentFileWriter; + private FileWriter queryPlanFileWriter; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; + private final boolean logQueryPlan; private final DatabaseProvider databaseProvider; private static final class AlsoWriteToConsoleFileWriter extends FileWriter { @@ -87,6 +90,10 @@ public StateLogger(String databaseName, DatabaseProvider provider, Main if (logEachSelect) { curFile = new File(dir, databaseName + "-cur.log"); } + logQueryPlan = options.logQueryPlan(); + if (logQueryPlan) { + queryPlanFile = new File(dir, databaseName + "-plan.log"); + } this.databaseProvider = provider; } @@ -138,6 +145,20 @@ public FileWriter getCurrentFileWriter() { return currentFileWriter; } + public FileWriter getQueryPlanFileWriter() { + if (!logQueryPlan) { + throw new UnsupportedOperationException(); + } + if (queryPlanFileWriter == null) { + try { + queryPlanFileWriter = new FileWriter(queryPlanFile, true); + } catch (IOException e) { + throw new AssertionError(e); + } + } + return queryPlanFileWriter; + } + public void writeCurrent(StateToReproduce state) { if (!logEachSelect) { throw new UnsupportedOperationException(); @@ -172,6 +193,18 @@ private void write(Loggable loggable) { } } + public void writeQueryPlan(String queryPlan) { + if (!logQueryPlan) { + throw new UnsupportedOperationException(); + } + try { + getQueryPlanFileWriter().append(removeNamesFromQueryPlans(queryPlan)); + queryPlanFileWriter.flush(); + } catch (IOException e) { + throw new AssertionError(); + } + } + public void logException(Throwable reduce, StateToReproduce state) { Loggable stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); @@ -201,8 +234,7 @@ private void printState(FileWriter writer, StateToReproduce state) { .getInfo(state.getDatabaseName(), state.getDatabaseVersion(), state.getSeedValue()).getLogString()); for (Query s : state.getStatements()) { - sb.append(s.getLogString()); - sb.append('\n'); + sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); } try { writer.write(sb.toString()); @@ -211,6 +243,13 @@ private void printState(FileWriter writer, StateToReproduce state) { } } + private String removeNamesFromQueryPlans(String queryPlan) { + String result = queryPlan; + result = result.replaceAll("t[0-9]+", "t0"); // Avoid duplicate tables + result = result.replaceAll("v[0-9]+", "v0"); // Avoid duplicate views + result = result.replaceAll("i[0-9]+", "i0"); // Avoid duplicate indexes + return result + "\n"; + } } public static class QueryManager { @@ -243,6 +282,10 @@ public void incrementSelectQueryCount() { Main.nrQueries.addAndGet(1); } + public Long getSelectQueryCount() { + return Main.nrQueries.get(); + } + public void incrementCreateDatabase() { Main.nrDatabases.addAndGet(1); } @@ -314,7 +357,12 @@ public void run() throws Exception { if (options.logEachSelect()) { logger.writeCurrent(state.getState()); } - Reproducer reproducer = provider.generateAndTestDatabase(state); + Reproducer reproducer = null; + if (options.enableQPG()) { + provider.generateAndTestDatabaseWithQueryPlanGuidance(state); + } else { + reproducer = provider.generateAndTestDatabase(state); + } try { logger.getCurrentFileWriter().close(); logger.currentFileWriter = null; diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 329010b39..5283ea2a6 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -50,6 +50,21 @@ public class MainOptions { @Parameter(names = "--print-failed", description = "Logs failed insert, create and other statements without results", arity = 1) private boolean loggerPrintFailed = true; // NOPMD + @Parameter(names = "--qpg-enable", description = "Enable the experimental feature Query Plan Guidance (QPG)", arity = 1) + private boolean enableQPG; + + @Parameter(names = "--qpg-log-query-plan", description = "Logs the query plans of each query (requires --qpg-enable)", arity = 1) + private boolean logQueryPlan; + + @Parameter(names = "--qpg-max-interval", description = "The maximum number of iterations to mutate tables if no new query plans (requires --qpg-enable)") + private static int qpgMaxInterval = 1000; + + @Parameter(names = "--qpg-reward-weight", description = "The weight (0-1) of last reward when updating weighted average reward. A higher value denotes average reward is more affected by the last reward (requires --qpg-enable)") + private static double qpgk = 0.25; + + @Parameter(names = "--qpg-selection-probability", description = "The probability (0-1) of the random selection of mutators. A higher value (>0.5) favors exploration over exploitation. (requires --qpg-enable)") + private static double qpgProbability = 0.7; + @Parameter(names = "--username", description = "The user name used to log into the DBMS") private String userName = "sqlancer"; // NOPMD @@ -151,6 +166,26 @@ public boolean loggerPrintFailed() { return loggerPrintFailed; } + public boolean logQueryPlan() { + return logQueryPlan; + } + + public boolean enableQPG() { + return enableQPG; + } + + public int getQPGMaxMutationInterval() { + return qpgMaxInterval; + } + + public double getQPGk() { + return qpgk; + } + + public double getQPGProbability() { + return qpgProbability; + } + public int getNrQueries() { return nrQueries; } diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 1a530e93e..47b15ee39 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -1,9 +1,14 @@ package sqlancer; +import java.sql.SQLException; +import java.util.HashMap; +import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import sqlancer.StateToReproduce.OracleRunReproductionState; +import sqlancer.common.DBMSCommon; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.schema.AbstractSchema; @@ -14,6 +19,13 @@ public abstract class ProviderAdapter globalClass; private final Class optionClass; + // Variables for QPG + Map queryPlanPool = new HashMap<>(); + static double[] weightedAverageReward; // static variable for sharing across all threads + int currentSelectRewards; + int currentSelectCounts; + int currentMutationOperator = -1; + public ProviderAdapter(Class globalClass, Class optionClass) { this.globalClass = globalClass; this.optionClass = optionClass; @@ -67,7 +79,7 @@ public Reproducer generateAndTestDatabase(G globalState) throws Exception { return null; } - protected abstract void checkViewsAreValid(G globalState); + protected abstract void checkViewsAreValid(G globalState) throws SQLException; protected TestOracle getTestOracle(G globalState) throws Exception { List> testOracleFactory = globalState.getDbmsSpecificOptions() @@ -77,7 +89,11 @@ protected TestOracle getTestOracle(G globalState) throws Exception { boolean userRequiresMoreThanZeroRows = globalState.getOptions().testOnlyWithMoreThanZeroRows(); boolean checkZeroRows = testOracleRequiresMoreThanZeroRows || userRequiresMoreThanZeroRows; if (checkZeroRows && globalState.getSchema().containsTableWithZeroRows(globalState)) { - throw new IgnoreMeException(); + if (globalState.getOptions().enableQPG()) { + addRowsToAllTables(globalState); + } else { + throw new IgnoreMeException(); + } } if (testOracleFactory.size() == 1) { return testOracleFactory.get(0).create(globalState); @@ -94,4 +110,152 @@ protected TestOracle getTestOracle(G globalState) throws Exception { public abstract void generateDatabase(G globalState) throws Exception; + // QPG: entry function + @Override + public void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws Exception { + if (weightedAverageReward == null) { + weightedAverageReward = initializeWeightedAverageReward(); // Same length as the list of mutators + } + try { + generateDatabase(globalState); + checkViewsAreValid(globalState); + globalState.getManager().incrementCreateDatabase(); + + Long executedQueryCount = 0L; + while (executedQueryCount < globalState.getOptions().getNrQueries()) { + int numOfNoNewQueryPlans = 0; + TestOracle oracle = getTestOracle(globalState); + while (true) { + try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { + assert localState != null; + try { + oracle.check(); + String query = oracle.getLastQueryString(); + executedQueryCount += 1; + if (addQueryPlan(query, globalState)) { + numOfNoNewQueryPlans = 0; + } else { + numOfNoNewQueryPlans++; + } + globalState.getManager().incrementSelectQueryCount(); + } catch (IgnoreMeException e) { + + } + assert localState != null; + localState.executedWithoutError(); + } + // exit loop to mutate tables if no new query plans have been found after a while + if (numOfNoNewQueryPlans > globalState.getOptions().getQPGMaxMutationInterval()) { + mutateTables(globalState); + break; + } + } + } + } finally { + globalState.getConnection().close(); + } + } + + // QPG: mutate tables for a new database state + private synchronized boolean mutateTables(G globalState) throws Exception { + // Update rewards based on a set of newly generated queries in last iteration + if (currentMutationOperator != -1) { + weightedAverageReward[currentMutationOperator] += ((double) currentSelectRewards + / (double) currentSelectCounts) * globalState.getOptions().getQPGk(); + } + currentMutationOperator = -1; + + // Choose mutator based on the rewards + int selectedActionIndex = 0; + if (Randomly.getPercentage() < globalState.getOptions().getQPGProbability()) { + selectedActionIndex = globalState.getRandomly().getInteger(0, weightedAverageReward.length); + } else { + selectedActionIndex = DBMSCommon.getMaxIndexInDoubleArrary(weightedAverageReward); + } + int reward = 0; + + try { + executeMutator(selectedActionIndex, globalState); + checkViewsAreValid(globalState); // Remove the invalid views + reward = checkQueryPlan(globalState); + } catch (IgnoreMeException | AssertionError e) { + } finally { + // Update rewards based on existing queries associated with the query plan pool + updateReward(selectedActionIndex, (double) reward / (double) queryPlanPool.size(), globalState); + currentMutationOperator = selectedActionIndex; + } + + // Clear the variables for storing the rewards of the action on a set of newly generated queries + currentSelectRewards = 0; + currentSelectCounts = 0; + return true; + } + + // QPG: add a query plan to the query plan pool and return true if the query plan is new + private boolean addQueryPlan(String selectStr, G globalState) throws Exception { + String queryPlan = getQueryPlan(selectStr, globalState); + + if (globalState.getOptions().logQueryPlan()) { + globalState.getLogger().writeQueryPlan(queryPlan); + } + + currentSelectCounts += 1; + if (queryPlanPool.containsKey(queryPlan)) { + return false; + } else { + queryPlanPool.put(queryPlan, selectStr); + currentSelectRewards += 1; + return true; + } + } + + // Obtain the reward of the current action based on the queries associated with the query plan pool + private int checkQueryPlan(G globalState) throws Exception { + int newQueryPlanFound = 0; + HashMap modifiedQueryPlan = new HashMap<>(); + for (Iterator> it = queryPlanPool.entrySet().iterator(); it.hasNext();) { + Map.Entry item = it.next(); + String queryPlan = item.getKey(); + String selectStr = item.getValue(); + String newQueryPlan = getQueryPlan(selectStr, globalState); + if (newQueryPlan.isEmpty()) { // Invalid query + it.remove(); + } else if (!queryPlan.equals(newQueryPlan)) { // A query plan has been changed + it.remove(); + modifiedQueryPlan.put(newQueryPlan, selectStr); + if (!queryPlanPool.containsKey(newQueryPlan)) { // A new query plan is found + newQueryPlanFound++; + } + } + } + queryPlanPool.putAll(modifiedQueryPlan); + return newQueryPlanFound; + } + + // QPG: update the reward of current action + private void updateReward(int actionIndex, double reward, G globalState) { + weightedAverageReward[actionIndex] += (reward - weightedAverageReward[actionIndex]) + * globalState.getOptions().getQPGk(); + } + + // QPG: initialize the weighted average reward of all mutation operators (required implementation in specific DBMS) + protected double[] initializeWeightedAverageReward() { + throw new UnsupportedOperationException(); + } + + // QPG: obtain the query plan of a query (required implementation in specific DBMS) + protected String getQueryPlan(String selectStr, G globalState) throws Exception { + throw new UnsupportedOperationException(); + } + + // QPG: execute a mutation operator (required implementation in specific DBMS) + protected void executeMutator(int index, G globalState) throws Exception { + throw new UnsupportedOperationException(); + } + + // QPG: add rows to all tables (required implementation in specific DBMS when enabling PQS oracle for QPG) + protected boolean addRowsToAllTables(G globalState) throws Exception { + throw new UnsupportedOperationException(); + } + } diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index a79d578db..9bc72b614 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -1,5 +1,6 @@ package sqlancer.cockroachdb; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -37,6 +38,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; @AutoService(DatabaseProvider.class) public class CockroachDBProvider extends SQLProviderAdapter { @@ -285,4 +287,57 @@ public String getDBMSName() { return "cockroachdb"; } + @Override + public String getQueryPlan(String selectStr, CockroachDBGlobalState globalState) throws Exception { + String queryPlan = ""; + String explainQuery = "EXPLAIN (OPT) " + selectStr; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, null); + boolean afterProjection = false; // Remove the concrete expression after each Projection operator + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String targetQueryPlan = rs.getString(1).replace("└──", "").replace("├──", "").replace("│", "") + .trim() + ";"; // Unify format + if (afterProjection) { + afterProjection = false; + continue; + } + if (targetQueryPlan.startsWith("projections")) { + afterProjection = true; + } + // Remove all concrete expressions by keywords + if (targetQueryPlan.contains(">") || targetQueryPlan.contains("<") || targetQueryPlan.contains("=") + || targetQueryPlan.contains("*") || targetQueryPlan.contains("+") + || targetQueryPlan.contains("'")) { + continue; + } + queryPlan += targetQueryPlan; + } + } + } catch (AssertionError e) { + throw new AssertionError("Explain failed: " + explainQuery); + } + + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, CockroachDBGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + } diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java index 2c7531d10..122b34a83 100644 --- a/src/sqlancer/common/DBMSCommon.java +++ b/src/sqlancer/common/DBMSCommon.java @@ -27,4 +27,16 @@ public static boolean matchesIndexName(String indexName) { return matcher.matches(); } + public static int getMaxIndexInDoubleArrary(double... doubleArray) { + int maxIndex = 0; + double maxValue = 0.0; + for (int j = 0; j < doubleArray.length; j++) { + double curReward = doubleArray[j]; + if (curReward > maxValue) { + maxIndex = j; + maxValue = curReward; + } + } + return maxIndex; + } } diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index 1cb5e0633..b20701c95 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -1,11 +1,13 @@ package sqlancer.sqlite3; import java.io.File; +import java.io.IOException; import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -20,6 +22,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; import sqlancer.sqlite3.gen.SQLite3CreateVirtualRtreeTabelGenerator; @@ -299,4 +302,59 @@ public SQLConnection createDatabase(SQLite3GlobalState globalState) throws SQLEx public String getDBMSName() { return "sqlite3"; } + + @Override + public String getQueryPlan(String selectStr, SQLite3GlobalState globalState) throws Exception { + String queryPlan = ""; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(selectStr); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + // Set up the expected errors for NoREC oracle. + ExpectedErrors errors = new ExpectedErrors(); + SQLite3Errors.addExpectedExpressionErrors(errors); + SQLite3Errors.addMatchQueryErrors(errors); + SQLite3Errors.addQueryErrors(errors); + SQLite3Errors.addInsertUpdateErrors(errors); + + SQLQueryAdapter q = new SQLQueryAdapter(SQLite3ExplainGenerator.explain(selectStr), errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + queryPlan += rs.getString(4) + ";"; + } + } + } catch (SQLException | AssertionError e) { + queryPlan = ""; + } + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, SQLite3GlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + + } + + @Override + protected boolean addRowsToAllTables(SQLite3GlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (SQLite3Table table : tablesNoRow) { + SQLQueryAdapter queryAddRows = SQLite3InsertGenerator.insertRow(globalState, table); + globalState.executeStatement(queryAddRows); + } + + return true; + } } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index dd02a474f..fbb905eea 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -1,5 +1,6 @@ package sqlancer.tidb; +import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.SQLException; @@ -18,6 +19,7 @@ import sqlancer.StatementExecutor; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; @@ -169,4 +171,43 @@ public String getDBMSName() { return "tidb"; } + @Override + public String getQueryPlan(String selectStr, TiDBGlobalState globalState) throws Exception { + String queryPlan = ""; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(selectStr); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN " + selectStr, null); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String targetQueryPlan = rs.getString(1).replace("├─", "").replace("└─", "").replace("│", "").trim() + + ";"; // Unify format + queryPlan += targetQueryPlan; + } + } + } catch (Throwable e) { + e.printStackTrace(); + } + + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, TiDBGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + } From 327ac2976608374b7a6e673ac057f8aad3d45dcb Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 19 Dec 2022 16:58:40 +0800 Subject: [PATCH 281/856] doc: add QPG (#663) --- README.md | 7 ++++++- docs/PAPERS.md | 16 +++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index af793f42c..283efcd29 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Man SQLancer operates in the following two phases: -1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. +1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. **New: we support query plan guided generation now. See Generation Approaches below.** 2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. # Getting Started @@ -40,6 +40,11 @@ If you launch SQLancer without parameters, available options and commands are di | Pivoted Query Synthesis (PQS) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described [here](https://arxiv.org/abs/2001.04174). PQS is the most powerful technique, but also requires more implementation effort than the other two techniques. It is currently unmaintained. | | Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | | Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | +# Generation Approaches +| Approach | Description | +|----------|-------------| +| Random Generation | Random generation is the default test case generation approach in SQLancer. First, random tables are generated. Then queries are randomly generated based on the schemas of the tables. | +| Query Plan Guidance (QPG) | QPG is a test case generation method guided by query plan coverage. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. This approach is enabled by option `--qpg-enable` and now supports TLP and NoREC oracles for SQLite, CockroachDB, and TiDB. | Please find the `.bib` entries [here](docs/PAPERS.md). diff --git a/docs/PAPERS.md b/docs/PAPERS.md index ca9d40de7..45f57e887 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -1,6 +1,6 @@ # Papers -The testing approaches implemented in SQLancer are described in the three papers below. +The testing approaches implemented in SQLancer are described in the four papers below. ## Testing Database Engines via Pivoted Query Synthesis @@ -51,6 +51,20 @@ This paper describes TLP, a metamorphic testing approach that can detect various } ``` +## Testing Database Engines via Query Plan Guidance + +This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. + +``` +@inproceedings{Ba2023QPG, + author = {Ba, Jinsheng and Rigger, Manuel}, + title = {Testing Database Engines via Query Plan Guidance}, + booktitle = {The 45th International Conference on Software Engineering (ICSE'23)}, + year = {2023}, + month = may +} +``` + # Comparing SQLancer With Other Tools that Find Logic Bugs If you want to fairly compare other tools with SQLancer, we would be glad to provide feedback (e.g., feel free to send an email to manuel.rigger@inf.ethz.ch). We have the following general recommendations and comments: From 7be4b6972858d048a9b1289d998491dd92cc39aa Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 19 Dec 2022 17:49:44 +0800 Subject: [PATCH 282/856] bypass new errors in SQLite (#664) --- src/sqlancer/sqlite3/SQLite3Errors.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 6cd4c40c9..6827116af 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -47,6 +47,7 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("json_insert() needs an odd number of arguments"); errors.add("json_object() labels must be TEXT"); errors.add("json_object() requires an even number of arguments"); + errors.add("argument of ntile must be a positive integer"); // fts5 functions errors.add("unable to use function highlight in the requested context"); @@ -61,6 +62,11 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("ORDER BY clause should come after"); errors.add("LIMIT clause should come after"); + errors.add("unsafe use of load_extension"); + errors.add("table does not support scanning"); + errors.add("circularly defined"); + errors.add("[SQLITE_ERROR] SQL error or missing database"); // A possible delay in the execution of DROP TABLE + // statement. } public static void addMatchQueryErrors(ExpectedErrors errors) { From d970f525c5edf8d4ae2d05c7581b71a844a3cd03 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 10:16:07 +0000 Subject: [PATCH 283/856] build(deps): bump slf4j-simple from 2.0.3 to 2.0.6 (#668) Bumps [slf4j-simple](https://github.com/qos-ch/slf4j) from 2.0.3 to 2.0.6. - [Release notes](https://github.com/qos-ch/slf4j/releases) - [Commits](https://github.com/qos-ch/slf4j/compare/v_2.0.3...v_2.0.6) --- updated-dependencies: - dependency-name: org.slf4j:slf4j-simple dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7b163a49f..1651566d3 100644 --- a/pom.xml +++ b/pom.xml @@ -325,7 +325,7 @@ org.slf4j slf4j-simple - 2.0.3 + 2.0.6 ru.yandex.clickhouse From 4d42dad02d0b6d43cac2a05e7e7741e6587bf626 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 19 Dec 2022 20:21:18 +0800 Subject: [PATCH 284/856] test cases for QPG (#670) * testcase for QPG in SQLite * workflows: QPG Tests (SQLite) --- .github/workflows/main.yml | 17 +++++++++++++++++ test/sqlancer/dbms/TestSQLiteQPG.java | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 test/sqlancer/dbms/TestSQLiteQPG.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a15ac75b..5b498033d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -272,6 +272,23 @@ jobs: mvn -Dtest=TestSQLitePQS test mvn -Dtest=TestSQLite3 test + sqlite-qpg: + name: QPG Tests (SQLite) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build + run: mvn -B package -DskipTests=true + - name: SQLite Tests for QPG + run: | + mvn -Dtest=TestSQLiteQPG test tidb: name: DBMS Tests (TiDB) diff --git a/test/sqlancer/dbms/TestSQLiteQPG.java b/test/sqlancer/dbms/TestSQLiteQPG.java new file mode 100644 index 000000000..37be6f78d --- /dev/null +++ b/test/sqlancer/dbms/TestSQLiteQPG.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLiteQPG { + + @Test + public void testSqliteQPG() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", "1000", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", + "--database-prefix", "pqsdb" /* Workaround for connections not being closed */, "--qpg-enable", "true", + "sqlite3", "--oracle", "NoREC", "--test-fts", "false", "--test-rtree", "false", + "--test-check-constraints", "false", "--test-in-operator", "false" })); + } + +} From 2fdc987f53f23812abe76e31b14ffc9c177b2155 Mon Sep 17 00:00:00 2001 From: Nathee Jaywaree <73115539+eltfshr@users.noreply.github.com> Date: Mon, 19 Dec 2022 21:12:18 +0700 Subject: [PATCH 285/856] Refactor UpdateGenerator classes using AbstractUpdateGenerator class (#662) * Add AbstractUpdateGenerator class * Refactor UpdateGenerator classes * Add ExpectedErrors to AbstractUpdateGenerator Co-authored-by: Manuel Rigger --- .../gen/CockroachDBUpdateGenerator.java | 34 +++++---- .../common/gen/AbstractUpdateGenerator.java | 26 +++++++ .../duckdb/gen/DuckDBUpdateGenerator.java | 49 ++++++------ src/sqlancer/h2/H2UpdateGenerator.java | 35 +++++---- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 53 +++++++------ .../gen/OceanBaseUpdateGenerator.java | 36 +++++---- .../postgres/gen/PostgresUpdateGenerator.java | 74 ++++++++++--------- .../gen/dml/SQLite3UpdateGenerator.java | 30 +++----- .../tidb/gen/TiDBUpdateGenerator.java | 45 ++++++----- .../ycql/gen/YCQLUpdateGenerator.java | 50 +++++++------ .../ysql/gen/YSQLUpdateGenerator.java | 73 +++++++++--------- 11 files changed, 276 insertions(+), 229 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractUpdateGenerator.java diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index b8a96e4c7..8dcd605d7 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -9,20 +9,27 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; -public final class CockroachDBUpdateGenerator { +public final class CockroachDBUpdateGenerator extends AbstractUpdateGenerator { - private CockroachDBUpdateGenerator() { + private final CockroachDBGlobalState globalState; + private CockroachDBExpressionGenerator gen; + + private CockroachDBUpdateGenerator(CockroachDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); + return new CockroachDBUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); - StringBuilder sb = new StringBuilder("UPDATE "); + gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); + sb.append("UPDATE "); sb.append(table.getName()); if (Randomly.getBoolean()) { sb.append("@{FORCE_INDEX="); @@ -30,15 +37,7 @@ public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { sb.append("}"); } sb.append(" SET "); - int i = 0; - for (CockroachDBColumn c : columns) { - if (i++ != 0) { - sb.append(", "); - } - sb.append(c.getName()); - sb.append("="); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(c.getType()))); - } + updateColumns(columns); if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); @@ -55,4 +54,9 @@ public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(CockroachDBColumn column) { + sb.append(CockroachDBVisitor.asString(gen.generateExpression(column.getType()))); + } + } diff --git a/src/sqlancer/common/gen/AbstractUpdateGenerator.java b/src/sqlancer/common/gen/AbstractUpdateGenerator.java new file mode 100644 index 000000000..f130c15a5 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractUpdateGenerator.java @@ -0,0 +1,26 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractUpdateGenerator> { + + protected final ExpectedErrors errors = new ExpectedErrors(); + protected StringBuilder sb = new StringBuilder(); + + protected void updateColumns(List columns) { + for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { + if (nrColumn != 0) { + sb.append(", "); + } + sb.append(columns.get(nrColumn).getName()); + sb.append("="); + updateValue(columns.get(nrColumn)); + } + } + + protected abstract void updateValue(C column); + +} diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index ba8c867fd..13b7f9d74 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -4,7 +4,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -13,36 +13,41 @@ import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -public final class DuckDBUpdateGenerator { +public final class DuckDBUpdateGenerator extends AbstractUpdateGenerator { - private DuckDBUpdateGenerator() { + private final DuckDBGlobalState globalState; + private DuckDBExpressionGenerator gen; + + private DuckDBUpdateGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); + return new DuckDBUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - Node expr; - if (Randomly.getBooleanWithSmallProbability()) { - expr = gen.generateExpression(); - DuckDBErrors.addExpressionErrors(errors); - } else { - expr = gen.generateConstant(); - } - sb.append(DuckDBToStringVisitor.asString(expr)); - } + updateColumns(columns); DuckDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(DuckDBColumn column) { + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + DuckDBErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(DuckDBToStringVisitor.asString(expr)); + } + } diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index 0dfc15938..158621409 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -3,33 +3,33 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2Table; -public final class H2UpdateGenerator { +public final class H2UpdateGenerator extends AbstractUpdateGenerator { - private H2UpdateGenerator() { + private final H2GlobalState globalState; + private H2ExpressionGenerator gen; + + private H2UpdateGenerator(H2GlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); + return new H2UpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new H2ExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); - H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - sb.append(H2ToStringVisitor.asString(gen.generateConstant())); - } + updateColumns(columns); H2Errors.addInsertErrors(errors); H2Errors.addDeleteErrors(errors); if (Randomly.getBoolean()) { @@ -40,4 +40,9 @@ public static SQLQueryAdapter getQuery(H2GlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(H2Column column) { + sb.append(H2ToStringVisitor.asString(gen.generateConstant())); + } + } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index d7e2ed1a4..bc1a220b1 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -4,54 +4,53 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBErrors; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBCompositeDataType; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBDataType; import sqlancer.hsqldb.HSQLDBToStringVisitor; import sqlancer.hsqldb.ast.HSQLDBExpression; -public final class HSQLDBUpdateGenerator { +public final class HSQLDBUpdateGenerator extends AbstractUpdateGenerator { - private static final ExpectedErrors EXPECTED_ERRORS = new ExpectedErrors(); private final HSQLDBProvider.HSQLDBGlobalState globalState; + private HSQLDBExpressionGenerator gen; private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { this.globalState = globalState; } public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { - return new HSQLDBUpdateGenerator(globalState).getQuery(); + return new HSQLDBUpdateGenerator(globalState).generate(); } - private SQLQueryAdapter getQuery() { - StringBuilder sb = new StringBuilder("UPDATE "); + private SQLQueryAdapter generate() { HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); - HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); - ExpectedErrors errors = new ExpectedErrors(); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - Node expr; - expr = gen.generateConstant(columns.get(i).getType()); - sb.append(HSQLDBToStringVisitor.asString(expr)); - if (Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); - expr = gen.generateExpression(columns.get(i).getType()); - sb.append(HSQLDBToStringVisitor.asString(expr)); - errors.add("data type of expression is not boolean"); - HSQLDBErrors.addExpressionErrors(errors); - } - + updateColumns(columns); + if (Randomly.getBooleanWithSmallProbability()) { + sb.append(" WHERE "); + sb.append(HSQLDBToStringVisitor.asString( + gen.generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)))); + errors.add("data type of expression is not boolean"); + HSQLDBErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), EXPECTED_ERRORS); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void updateValue(HSQLDBColumn column) { + Node expr; + expr = gen.generateConstant(column.getType()); + sb.append(HSQLDBToStringVisitor.asString(expr)); } } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index f89434498..950317bd2 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -3,17 +3,18 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; import sqlancer.oceanbase.OceanBaseVisitor; -public class OceanBaseUpdateGenerator { +public class OceanBaseUpdateGenerator extends AbstractUpdateGenerator { - private final StringBuilder sb = new StringBuilder(); private final OceanBaseGlobalState globalState; + private OceanBaseExpressionGenerator gen; private final Randomly r; public OceanBaseUpdateGenerator(OceanBaseGlobalState globalState) { @@ -26,29 +27,16 @@ public static SQLQueryAdapter update(OceanBaseGlobalState globalState) { } private SQLQueryAdapter generate() { - ExpectedErrors errors = new ExpectedErrors(); OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append("UPDATE "); if (Randomly.getBoolean()) { sb.append(" /*+parallel(" + r.getInteger(0, 10) + ") enable_parallel_dml*/ "); } sb.append(table.getName()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - if (Randomly.getBoolean()) { - sb.append(gen.generateConstant(columns.get(i))); - } else { - sb.append(OceanBaseVisitor.asString(gen.generateExpression())); - OceanBaseErrors.addExpressionErrors(errors); - } - } + updateColumns(columns); if (Randomly.getBoolean()) { sb.append(" WHERE "); OceanBaseErrors.addExpressionErrors(errors); @@ -60,4 +48,14 @@ private SQLQueryAdapter generate() { return new SQLQueryAdapter(sb.toString(), errors); } + + @Override + protected void updateValue(OceanBaseColumn column) { + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant(column)); + } else { + sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + OceanBaseErrors.addExpressionErrors(errors); + } + } } diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 1122b2b5a..7ce7fe882 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -1,9 +1,10 @@ package sqlancer.postgres.gen; +import java.util.Arrays; import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -12,51 +13,36 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public final class PostgresUpdateGenerator { +public final class PostgresUpdateGenerator extends AbstractUpdateGenerator { - private PostgresUpdateGenerator() { + private final PostgresGlobalState globalState; + private PostgresTable randomTable; + + private PostgresUpdateGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; + errors.addAll(Arrays.asList("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", + " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable")); } public static SQLQueryAdapter create(PostgresGlobalState globalState) { - PostgresTable randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); - StringBuilder sb = new StringBuilder(); + return new PostgresUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); sb.append(randomTable.getName()); sb.append(" SET "); - ExpectedErrors errors = ExpectedErrors.from("conflicting key value violates exclusion constraint", - "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", - "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", - " bit string too long", "can only be updated to DEFAULT", "division by zero", - "You might need to add explicit type casts.", "invalid regular expression", - "View columns that are not columns of their base relation are not updatable"); errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced // table multiple times errors.add("new row violates check option for view"); - List columns = randomTable.getRandomNonEmptyColumnSubset(); PostgresCommon.addCommonInsertUpdateErrors(errors); - - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - PostgresColumn column = columns.get(i); - sb.append(column.getName()); - sb.append(" = "); - if (!Randomly.getBoolean()) { - PostgresExpression constant = PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), - column.getType()); - sb.append(PostgresVisitor.asString(constant)); - } else if (Randomly.getBoolean()) { - sb.append("DEFAULT"); - } else { - sb.append("("); - PostgresExpression expr = PostgresExpressionGenerator.generateExpression(globalState, - randomTable.getColumns(), column.getType()); - // caused by casts - sb.append(PostgresVisitor.asString(expr)); - sb.append(")"); - } - } + updateColumns(columns); errors.add("invalid input syntax for "); errors.add("operator does not exist: text = boolean"); errors.add("violates check constraint"); @@ -73,4 +59,22 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors, true); } + @Override + protected void updateValue(PostgresColumn column) { + if (!Randomly.getBoolean()) { + PostgresExpression constant = PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(PostgresVisitor.asString(constant)); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append("("); + PostgresExpression expr = PostgresExpressionGenerator.generateExpression(globalState, + randomTable.getColumns(), column.getType()); + // caused by casts + sb.append(PostgresVisitor.asString(expr)); + sb.append(")"); + } + } + } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 791b7f5eb..5a17ad339 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -4,7 +4,7 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; @@ -14,12 +14,10 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3UpdateGenerator { +public class SQLite3UpdateGenerator extends AbstractUpdateGenerator { - private final StringBuilder sb = new StringBuilder(); - private final Randomly r; - private final ExpectedErrors errors = new ExpectedErrors(); private final SQLite3GlobalState globalState; + private final Randomly r; public SQLite3UpdateGenerator(SQLite3GlobalState globalState, Randomly r) { this.globalState = globalState; @@ -34,10 +32,11 @@ public static SQLQueryAdapter updateRow(SQLite3GlobalState globalState) { public static SQLQueryAdapter updateRow(SQLite3GlobalState globalState, SQLite3Table table) { SQLite3UpdateGenerator generator = new SQLite3UpdateGenerator(globalState, globalState.getRandomly()); - return generator.update(table); + return generator.generate(table); } - private SQLQueryAdapter update(SQLite3Table table) { + private SQLQueryAdapter generate(SQLite3Table table) { + List columnsToUpdate = Randomly.nonEmptySubsetPotentialDuplicates(table.getColumns()); sb.append("UPDATE "); if (Randomly.getBoolean()) { sb.append("OR IGNORE "); @@ -55,7 +54,6 @@ private SQLQueryAdapter update(SQLite3Table table) { sb.append(table.getName()); sb.append(" SET "); - List columnsToUpdate = Randomly.nonEmptySubsetPotentialDuplicates(table.getColumns()); if (Randomly.getBoolean()) { sb.append("("); sb.append(columnsToUpdate.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); @@ -66,19 +64,12 @@ private SQLQueryAdapter update(SQLite3Table table) { if (i != 0) { sb.append(", "); } - getToUpdateValue(columnsToUpdate, i); + updateValue(columnsToUpdate.get(i)); } sb.append(")"); // row values } else { - for (int i = 0; i < columnsToUpdate.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columnsToUpdate.get(i).getName()); - sb.append(" = "); - getToUpdateValue(columnsToUpdate, i); - } + updateColumns(columnsToUpdate); } if (Randomly.getBoolean()) { @@ -111,8 +102,9 @@ private SQLQueryAdapter update(SQLite3Table table) { } - private void getToUpdateValue(List columnsToUpdate, int i) { - if (columnsToUpdate.get(i).isIntegerPrimaryKey()) { + @Override + protected void updateValue(SQLite3Column column) { + if (column.isIntegerPrimaryKey()) { sb.append(SQLite3Visitor.asString(SQLite3Constant.createIntConstant(r.getInteger()))); } else { sb.append(SQLite3Visitor.asString(SQLite3ExpressionGenerator.getRandomLiteralValue(globalState))); diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 4912899fe..241ee3321 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -4,7 +4,7 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; @@ -13,32 +13,27 @@ import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.visitor.TiDBVisitor; -public final class TiDBUpdateGenerator { +public final class TiDBUpdateGenerator extends AbstractUpdateGenerator { - private TiDBUpdateGenerator() { + private final TiDBGlobalState globalState; + private TiDBExpressionGenerator gen; + + private TiDBUpdateGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - ExpectedErrors errors = new ExpectedErrors(); + return new TiDBUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() throws SQLException { TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); - StringBuilder sb = new StringBuilder("UPDATE "); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - if (Randomly.getBoolean()) { - sb.append(gen.generateConstant()); - } else { - sb.append(TiDBVisitor.asString(gen.generateExpression())); - TiDBErrors.addExpressionErrors(errors); - } - } + updateColumns(columns); if (Randomly.getBoolean()) { sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); @@ -49,4 +44,14 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(TiDBColumn column) { + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant()); + } else { + sb.append(TiDBVisitor.asString(gen.generateExpression())); + TiDBErrors.addExpressionErrors(errors); + } + } + } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java index d417ca96a..695301595 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -4,7 +4,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLErrors; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; @@ -13,35 +13,27 @@ import sqlancer.yugabyte.ycql.YCQLToStringVisitor; import sqlancer.yugabyte.ycql.ast.YCQLExpression; -public final class YCQLUpdateGenerator { +public final class YCQLUpdateGenerator extends AbstractUpdateGenerator { - private YCQLUpdateGenerator() { + private final YCQLGlobalState globalState; + private YCQLExpressionGenerator gen; + + private YCQLUpdateGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - StringBuilder sb = new StringBuilder("UPDATE "); - ExpectedErrors errors = new ExpectedErrors(); + return new YCQLUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); sb.append(table.getName()); - YCQLExpressionGenerator gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append(" SET "); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append("="); - Node expr; - if (Randomly.getBooleanWithSmallProbability()) { - expr = gen.generateExpression(); - YCQLErrors.addExpressionErrors(errors); - } else { - expr = gen.generateConstant(); - } - sb.append(YCQLToStringVisitor.asString(expr)); - } - + updateColumns(columns); errors.add("Invalid Arguments"); errors.add("Invalid CQL Statement"); errors.add("Invalid SQL Statement"); @@ -53,4 +45,16 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors); } + @Override + protected void updateValue(YCQLColumn column) { + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(); + YCQLErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(); + } + sb.append(YCQLToStringVisitor.asString(expr)); + } + } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index 06e762440..bc7b00d79 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -1,9 +1,10 @@ package sqlancer.yugabyte.ysql.gen; +import java.util.Arrays; import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; @@ -13,51 +14,37 @@ import sqlancer.yugabyte.ysql.YSQLVisitor; import sqlancer.yugabyte.ysql.ast.YSQLExpression; -public final class YSQLUpdateGenerator { +public final class YSQLUpdateGenerator extends AbstractUpdateGenerator { - private YSQLUpdateGenerator() { + private final YSQLGlobalState globalState; + private YSQLTable randomTable; + + private YSQLUpdateGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; + errors.addAll(Arrays.asList("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", + " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable")); } public static SQLQueryAdapter create(YSQLGlobalState globalState) { - YSQLTable randomTable = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); - StringBuilder sb = new StringBuilder(); + return new YSQLUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + randomTable = globalState.getSchema().getRandomTable(YSQLTable::isInsertable); + List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); sb.append(randomTable.getName()); sb.append(" SET "); - ExpectedErrors errors = ExpectedErrors.from("conflicting key value violates exclusion constraint", - "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", - "violates unique constraint", "out of range", "cannot cast", "must be type boolean", "is not unique", - " bit string too long", "can only be updated to DEFAULT", "division by zero", - "You might need to add explicit type casts.", "invalid regular expression", - "View columns that are not columns of their base relation are not updatable"); errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced // table multiple times errors.add("new row violates check option for view"); - List columns = randomTable.getRandomNonEmptyColumnSubset(); YSQLErrors.addCommonInsertUpdateErrors(errors); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - YSQLColumn column = columns.get(i); - sb.append(column.getName()); - sb.append(" = "); - if (!Randomly.getBoolean()) { - YSQLExpression constant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), - column.getType()); - sb.append(YSQLVisitor.asString(constant)); - } else if (Randomly.getBoolean()) { - sb.append("DEFAULT"); - } else { - sb.append("("); - YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), - column.getType()); - // caused by casts - sb.append(YSQLVisitor.asString(expr)); - sb.append(")"); - } - } + updateColumns(columns); errors.add("invalid input syntax for "); errors.add("operator does not exist: text = boolean"); errors.add("violates check constraint"); @@ -74,4 +61,22 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), errors, true); } + @Override + protected void updateValue(YSQLColumn column) { + if (!Randomly.getBoolean()) { + YSQLExpression constant = YSQLExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(YSQLVisitor.asString(constant)); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append("("); + YSQLExpression expr = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), + column.getType()); + // caused by casts + sb.append(YSQLVisitor.asString(expr)); + sb.append(")"); + } + } + } From f6e7c608a5c42c1034ed71845d34931f8add826b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 14:34:46 +0000 Subject: [PATCH 286/856] build(deps): bump maven-dependency-plugin from 3.3.0 to 3.4.0 (#667) Bumps [maven-dependency-plugin](https://github.com/apache/maven-dependency-plugin) from 3.3.0 to 3.4.0. - [Release notes](https://github.com/apache/maven-dependency-plugin/releases) - [Commits](https://github.com/apache/maven-dependency-plugin/compare/maven-dependency-plugin-3.3.0...maven-dependency-plugin-3.4.0) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-dependency-plugin dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1651566d3..1e6ea73dc 100644 --- a/pom.xml +++ b/pom.xml @@ -154,7 +154,7 @@ org.apache.maven.plugins maven-dependency-plugin - 3.3.0 + 3.4.0 copy-dependencies From bf08f1d7ed2f1f34d152bff2968577b3dc6f8c07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 14:58:00 +0000 Subject: [PATCH 287/856] build(deps): bump plexus-compiler-api from 2.12.1 to 2.13.0 (#666) Bumps [plexus-compiler-api](https://github.com/codehaus-plexus/plexus-compiler) from 2.12.1 to 2.13.0. - [Release notes](https://github.com/codehaus-plexus/plexus-compiler/releases) - [Commits](https://github.com/codehaus-plexus/plexus-compiler/compare/plexus-compiler-2.12.1...plexus-compiler-2.13.0) --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1e6ea73dc..320e24897 100644 --- a/pom.xml +++ b/pom.xml @@ -133,7 +133,7 @@ org.codehaus.plexus plexus-compiler-api - 2.12.1 + 2.13.0 From c5ab213a4f997d66fdd6b0adf7f5ab07d1b83a33 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 20 Dec 2022 03:14:19 +0800 Subject: [PATCH 288/856] build(deps): bump plexus-compiler-eclipse from 2.12.1 to 2.13.0 (#665) Bumps plexus-compiler-eclipse from 2.12.1 to 2.13.0. --- updated-dependencies: - dependency-name: org.codehaus.plexus:plexus-compiler-eclipse dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 320e24897..b61126106 100644 --- a/pom.xml +++ b/pom.xml @@ -123,7 +123,7 @@ org.codehaus.plexus plexus-compiler-eclipse - 2.12.1 + 2.13.0 org.eclipse.jdt From 30f8ac57b22a711ec75d4159c3c36096fcf8478b Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 20 Dec 2022 20:35:15 +0800 Subject: [PATCH 289/856] doc: QPG mutator list (#671) --- docs/QueryPlanGuidance.md | 66 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 docs/QueryPlanGuidance.md diff --git a/docs/QueryPlanGuidance.md b/docs/QueryPlanGuidance.md new file mode 100644 index 000000000..bb467461b --- /dev/null +++ b/docs/QueryPlanGuidance.md @@ -0,0 +1,66 @@ +# Query Plan Guidance +Query Plan Guidance (QPG) is a test case generation method that attempts to explore unseen query plans. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. Here, we document all mutators in which we choose the most promising one that may help covering more unique query plans to execute. + +# Mutators +All mutators are listed below and implemented in the enumeration variables `Action` in the `XXDBProvider.java` file of each DBMS. +The `Mutator` column includes the items in the `Action` enumeration variable. +The `Example` column includes an example of a realistic statement generated by this mutator. +The `Description` column includes an explanation of what the mutator does. +The `More unique query plans...` column explains why applying this mutator may help covering more unique query plans. + + +|DBMS |Mutator |Example |Description |More unique query plans may be covered because of | +|-----------|---------------------|--------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------|--------------------------------------------------------| +|SQLite |PRAGMA |PRAGMA automatic_index true; |It modifies database options. |different options that decide how to execute statements.| +|SQLite |CREATE_INDEX |CREATE INDEX i0 ON t0 WHERE c0 ISNULL; |It adds a new index on a table. |subsequent differnt logic of querying data. | +|SQLite |CREATE_VIEW |CREATE VIEW v0(c0) AS SELECT DISTINCT ABS(t0.c2) FROM t0; |It adds a new view from existing tables. |more possible execution logics on the view. | +|SQLite |CREATE_TABLE |CREATE TABLE t0 (c0 INT CHECK ((c0) BETWEEN (1) AND (10)) ); |It adds a new table. |more possible execution logics on the table. | +|SQLite |CREATE_VIRTUALTABLE |CREATE VIRTUAL TABLE vt1 USING fts5(c0 UNINDEXED); |It adds a new table with fts5 feature. |more possible execution logics on the table with fts5. | +|SQLite |CREATE_RTREETABLE |CREATE VIRTUAL TABLE rt0 USING rtree_i32(c0, c1, c2, c3, c4); |It adds a new table with rtree feature. |more possible execution logics on the table with rtree. | +|SQLite |INSERT |INSERT INTO t0(c0, c1) VALUES ('lrd+a*', NULL); |It inserts a new row to a table. |subsequent different logic of querying data. | +|SQLite |DELETE |DELETE FROM t0 WHERE (c0>3); |It deletes specific rows from a table. |subsequent different logic of querying data. | +|SQLite |ALTER |ALTER TABLE t0 ADD COLUMN c39 REAL; |It changes the schema of a table. |more possible execution logics on the changed table. | +|SQLite |UPDATE |UPDATE t0 SET (c2, c0)=(-944, 'L((xA') WHERE t0.c1; |It updates specific data of a table. |subsequent different logic of querying data. | +|SQLite |DROP_INDEX |DROP INDEX i0; |It drops an index. |subsequent different logic of querying data. | +|SQLite |DROP_TABLE |DROP TABLE t0; |it drops an table. |subsequent different logic of querying data. | +|SQLite |DROP_VIEW |DROP VIEW v0; |It drops a view. |subsequent different logic of querying data. | +|SQLite |VACUUM |VACUUM main; |It rebuilds the database file. |subsequent different logic of querying data. | +|SQLite |REINDEX |REINDEX t0; |It drops and recreates indexes from scratch. |subsequent different logic of querying data. | +|SQLite |ANALYZE |ANALYZE t0; |It gathers statistics about tables to help make better query planning choices.|subsequent different logic of querying data. | +|SQLite |EXPLAIN |EXPLAIN SELECT * FROM t0; |It obtains query plan of a query. |subsequent different logic of querying data. | +|SQLite |CHECK_RTREE_TABLE |SELECT rtreecheck('rt0'); |It runs an integrity check on a table. |subsequent different logic of querying data. | +|SQLite |VIRTUAL_TABLE_ACTION |INSERT INTO vt0(vt0) VALUES('rebuild'); |It changes the options of a virtual table. |subsequent different logic of querying data. | +|SQLite |MANIPULATE_STAT_TABLE|INSERT INTO sqlite_stat1 VALUES('rt0', 't1', '2'); |It changes the table that stores statistics of all tables. |subsequent different logic of querying data. | +|SQLite |TRANSACTION_START |BEGIN TRANSACTION; |All statements after this will not be committed. |subsequent different logic of querying data. | +|SQLite |ROLLBACK_TRANSACTION |ROLLBACK TRANSACTION; |All statements after last BEGIN are dropped. |subsequent different logic of querying data. | +|SQLite |COMMIT |COMMIT; |All statements after last BEGIN are committed |subsequent different logic of querying data. | +|TiDB |CREATE_TABLE |CREATE TABLE t1(c0 INT); |It adds a new table. |more possible execution logics on the table. | +|TiDB |CREATE_INDEX |CREATE INDEX i0 ON t0(c0(250) ASC) KEY_BLOCK_SIZE 1564693810209727437; |It adds a new index on a table. |subsequent differnt logic of querying data. | +|TiDB |VIEW_GENERATOR |CREATE VIEW v0(c0, c1) AS SELECT t1.c0, ((t1.c0)REGEXP('8')) FROM t1; |It adds a new view from existing tables. |more possible execution logics on the view. | +|TiDB |INSERT |INSERT INTO t0(c0) VALUES (-16387); |It inserts a new row to a table. |subsequent different logic of querying data. | +|TiDB |ALTER_TABLE |ALTER TABLE t1 ADD PRIMARY KEY(c0); |It changes the schema of a table. |more possible execution logics on the changed table. | +|TiDB |TRUNCATE |TRUNCATE t0; |It drops all rows of a table. |subsequent different logic of querying data. | +|TiDB |UPDATE |UPDATE t0 SET c0='S' WHERE t0.c0; |It updates specific data of a table. |subsequent different logic of querying data. | +|TiDB |DELETE |DELETE FROM t0 ORDER BY CAST(t0.c0 AS CHAR) DESC; |It deletes specific rows from a table. |subsequent different logic of querying data. | +|TiDB |SET |set @@tidb_max_chunk_size=8864; |It modifies database options. |different options that decide how to execute statements.| +|TiDB |ADMIN_CHECKSUM_TABLE |ADMIN CHECKSUM TABLE t0; |it calculate the checksum for a table. |subsequent different logic of querying data. | +|TiDB |ANALYZE_TABLE |ANALYZE TABLE t1 WITH 174 BUCKETS; |It gathers statistics about tables to help make better query planning choices.|subsequent different logic of querying data. | +|TiDB |DROP_TABLE |DROP TABLE t0; |it drops an table. |subsequent different logic of querying data. | +|TiDB |DROP_VIEW |DROP VIEW v0; |It drops a view. |subsequent different logic of querying data. | +|CockroachDB|CREATE_TABLE |CREATE TABLE t1 (c0 INT4, c1 VARBIT(44) UNIQUE DEFAULT (B'000'), CONSTRAINT "primary" PRIMARY KEY(c1 ASC, c0 ASC));|It adds a new table. |more possible execution logics on the table. | +|CockroachDB|CREATE_INDEX |CREATE INDEX ON t0(rowid); |It adds a new index on a table. |subsequent differnt logic of querying data. | +|CockroachDB|CREATE_VIEW |CREATE VIEW v0(c0) AS SELECT DISTINCT MIN(TIMETZ '1970-01-11T12:19:44') FROM t0; |It adds a new view from existing tables. |more possible execution logics on the view. | +|CockroachDB|CREATE_STATISTICS |CREATE STATISTICS s0 FROM t2; |It gathers statistics about tables to help make better query planning choices.|subsequent different logic of querying data. | +|CockroachDB|INSERT |INSERT INTO t1 (rowid, c0) VALUES(NULL, true) ON CONFLICT (c0) DO NOTHING ; |It inserts a new row to a table. |subsequent different logic of querying data. | +|CockroachDB|UPDATE |UPDATE t0@{FORCE_INDEX=t0_pkey} SET c0=t0.c0; |It updates specific data of a table. |subsequent different logic of querying data. | +|CockroachDB|SET_SESSION |SET SESSION BYTEA_OUTPUT=escape; |It changes session configurations. |different options that decide how to execute statements.| +|CockroachDB|SET_CLUSTER_SETTING |SET CLUSTER SETTING sql.query_cache.enabled=true; |It changes cluster configurations. |different options that decide how to execute statements.| +|CockroachDB|DELETE |DELETE from t0; |It deletes specific rows from a table. |subsequent different logic of querying data. | +|CockroachDB|TRUNCATE |TRUNCATE TABLE t1 CASCADE; |It drops all rows of a table. |subsequent different logic of querying data. | +|CockroachDB|DROP_TABLE |DROP TABLE t0; |it drops an table. |subsequent different logic of querying data. | +|CockroachDB|DROP_VIEW |DROP VIEW v0; |It drops a view. |subsequent different logic of querying data. | +|CockroachDB|COMMENT_ON |COMMENT ON INDEX t0_c0_key IS '|?'; |It changes schema of a table. |subsequent different logic of querying data. | +|CockroachDB|SHOW |SHOW LOCALITY; |It lists detailed information of active queries. |subsequent different logic of querying data. | +|CockroachDB|EXPLAIN |EXPLAIN SELECT * FROM t0; |It obtains query plan of a query. |subsequent different logic of querying data. | +|CockroachDB|SCRUB |EXPERIMENTAL SCRUB table t0; |It checks data corruption of a table. |subsequent different logic of querying data. | +|CockroachDB|SPLIT |ALTER INDEX t0@t0_c0_key SPLIT AT VALUES (NULL); |It changes the indexes. |subsequent different logic of querying data. | From 7ca7d36a395bc751c460419bc63f08499add3c4c Mon Sep 17 00:00:00 2001 From: Albert Zhang <72067599+albertZhangTJ@users.noreply.github.com> Date: Thu, 22 Dec 2022 22:43:24 +0800 Subject: [PATCH 290/856] Add regex support for ExpectedError (#672) * Add Regex support in ExpectedErrors * Add Regex support in ExpectedErrors * Add Regex support in ExpectedErrors * Add unit test cases for regex in ExpectedError * Revert "Add unit test cases for regex in ExpectedError" This reverts commit cc9fce209a5e59f76ed1f72f1857565f90849dc6. * Add unit test cases for regex in ExpectedError * Removed config file from PR * Removed config file from PR * Remove unused imports * Fix PMD violation * Remove usingXXX fields in ExpectedError * Remove test cases for usingXXX fields in ExpectedError * Remove test cases for usingXXX fields in ExpectedError Co-authored-by: Manuel Rigger --- src/sqlancer/common/query/ExpectedErrors.java | 53 ++++++++++++++----- test/sqlancer/TestExpectedErrors.java | 29 ++++++++-- 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index 20a7d97c8..1016969c9 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -1,8 +1,11 @@ package sqlancer.common.query; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; +import java.util.List; import java.util.Set; +import java.util.regex.Pattern; /** * This class represents the errors that executing a statement might result in. For example, an INSERT statement might @@ -12,6 +15,7 @@ public class ExpectedErrors { private final Set errors = new HashSet<>(); + private final List regexes = new ArrayList<>(); public ExpectedErrors add(String error) { if (error == null) { @@ -21,6 +25,35 @@ public ExpectedErrors add(String error) { return this; } + public ExpectedErrors addRegex(Pattern errorPattern) { + if (errorPattern == null) { + throw new IllegalArgumentException(); + } + regexes.add(errorPattern); + return this; + } + + public ExpectedErrors addAll(Collection list) { + errors.addAll(list); + return this; + } + + public ExpectedErrors addAllRegexes(Collection list) { + if (list == null) { + throw new IllegalArgumentException(); + } + regexes.addAll(list); + return this; + } + + public static ExpectedErrors from(String... errors) { + ExpectedErrors expectedErrors = new ExpectedErrors(); + for (String error : errors) { + expectedErrors.add(error); + } + return expectedErrors; + } + /** * Checks whether the error message (e.g., returned by the DBMS under test) contains any of the added error * messages. @@ -34,25 +67,17 @@ public boolean errorIsExpected(String error) { if (error == null) { throw new IllegalArgumentException(); } - for (String s : errors) { + for (String s : this.errors) { if (error.contains(s)) { return true; } } - return false; - } - - public ExpectedErrors addAll(Collection list) { - errors.addAll(list); - return this; - } - - public static ExpectedErrors from(String... errors) { - ExpectedErrors expectedErrors = new ExpectedErrors(); - for (String error : errors) { - expectedErrors.add(error); + for (Pattern p : this.regexes) { + if (p.matcher(error).find()) { + return true; + } } - return expectedErrors; + return false; } } diff --git a/test/sqlancer/TestExpectedErrors.java b/test/sqlancer/TestExpectedErrors.java index 69a0f53e6..2781df89b 100644 --- a/test/sqlancer/TestExpectedErrors.java +++ b/test/sqlancer/TestExpectedErrors.java @@ -1,5 +1,7 @@ package sqlancer; +import java.util.regex.Pattern; + import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -16,7 +18,7 @@ public void testEmpty() { } @Test - public void testSimple() { + public void testStringSimple() { ExpectedErrors errors = new ExpectedErrors(); errors.add("a"); errors.add("b"); @@ -25,16 +27,37 @@ public void testSimple() { assertTrue(errors.errorIsExpected("b")); assertTrue(errors.errorIsExpected("c")); assertTrue(errors.errorIsExpected("aa")); - assertFalse(errors.errorIsExpected("d")); + + } + + @Test + public void testRegexSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addRegex(Pattern.compile("a\\d")); + errors.addRegex(Pattern.compile("b\\D")); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } @Test - public void testRealistic() { + public void testStringRealistic() { ExpectedErrors errors = new ExpectedErrors(); errors.add("violated"); assertTrue(errors.errorIsExpected("UNIQUE constraint was violated!")); assertTrue(errors.errorIsExpected("PRIMARY KEY constraint was violated!")); } + @Test + public void testRegexRealistic() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addRegex(Pattern.compile(".violated.")); + assertTrue(errors.errorIsExpected("UNIQUE constraint was violated!")); + assertTrue(errors.errorIsExpected("PRIMARY KEY constraint was violated!")); + } + } From f41520312d8fba0dd31389b8d3d049cba5a54f38 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 24 Dec 2022 15:38:43 +0800 Subject: [PATCH 291/856] fix null pointers (#673) --- src/sqlancer/cockroachdb/CockroachDBProvider.java | 2 +- src/sqlancer/tidb/TiDBProvider.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 9bc72b614..c9e0d24a9 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -299,7 +299,7 @@ public String getQueryPlan(String selectStr, CockroachDBGlobalState globalState) e.printStackTrace(); } } - SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, null); + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery); boolean afterProjection = false; // Remove the concrete expression after each Projection operator try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs != null) { diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index fbb905eea..c4f13a207 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -183,7 +183,7 @@ public String getQueryPlan(String selectStr, TiDBGlobalState globalState) throws } } - SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN " + selectStr, null); + SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN " + selectStr); try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs != null) { while (rs.next()) { From 2a5ca983c4ded3b1042b405950637265f5388d12 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 24 Dec 2022 16:13:02 +0800 Subject: [PATCH 292/856] More tests for QPG (#674) * add tests for QPG * add actions for QPG --- .github/workflows/main.yml | 48 +++++++++++++++++- test/sqlancer/dbms/TestSQLiteQPG.java | 20 -------- .../qpg/cockroachdb/TestCockroachDBQPG.java | 24 +++++++++ .../cockroachdb/TestCockroachDBQueryPlan.java | 47 ++++++++++++++++++ test/sqlancer/qpg/sqlite/TestSQLiteQPG.java | 23 +++++++++ .../qpg/sqlite/TestSQLiteQueryPlan.java | 41 ++++++++++++++++ test/sqlancer/qpg/tidb/TestTiDBQPG.java | 24 +++++++++ test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java | 49 +++++++++++++++++++ 8 files changed, 255 insertions(+), 21 deletions(-) delete mode 100644 test/sqlancer/dbms/TestSQLiteQPG.java create mode 100644 test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java create mode 100644 test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java create mode 100644 test/sqlancer/qpg/sqlite/TestSQLiteQPG.java create mode 100644 test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java create mode 100644 test/sqlancer/qpg/tidb/TestTiDBQPG.java create mode 100644 test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5b498033d..b810c925b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -116,7 +116,30 @@ jobs: run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - + + cockroachdb-qpg: + name: QPG Tests (CockroachDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up CockroachDB + run: | + wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.0-beta.4.linux-amd64.tgz | tar xvz + cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach start-single-node --insecure & + sleep 10 + - name: Create SQLancer user + run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + - name: Run Tests + run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test + databend: name: DBMS Tests (Databend) runs-on: ubuntu-latest @@ -313,6 +336,29 @@ jobs: - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test + tidb-qpg: + name: QPG Tests (TiDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up TiDB + run: | + docker pull pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + sleep 10 + - name: Create SQLancer user + run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + - name: Run Tests + run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test + yugabyte: name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest diff --git a/test/sqlancer/dbms/TestSQLiteQPG.java b/test/sqlancer/dbms/TestSQLiteQPG.java deleted file mode 100644 index 37be6f78d..000000000 --- a/test/sqlancer/dbms/TestSQLiteQPG.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestSQLiteQPG { - - @Test - public void testSqliteQPG() { - assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", "1000", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", - "--database-prefix", "pqsdb" /* Workaround for connections not being closed */, "--qpg-enable", "true", - "sqlite3", "--oracle", "NoREC", "--test-fts", "false", "--test-rtree", "false", - "--test-check-constraints", "false", "--test-in-operator", "false" })); - } - -} diff --git a/test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java new file mode 100644 index 000000000..b3ba57a10 --- /dev/null +++ b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQPG.java @@ -0,0 +1,24 @@ +package sqlancer.qpg.cockroachdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestCockroachDBQPG { + + @Test + public void testCockroachDBQPG() { + String cockroachDB = System.getenv("COCKROACHDB_AVAILABLE"); + boolean cockroachDBIsAvailable = cockroachDB != null && cockroachDB.equalsIgnoreCase("true"); + assumeTrue(cockroachDBIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, + "cockroachdb", "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java new file mode 100644 index 000000000..c858ee190 --- /dev/null +++ b/test/sqlancer/qpg/cockroachdb/TestCockroachDBQueryPlan.java @@ -0,0 +1,47 @@ +package sqlancer.qpg.cockroachdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.cockroachdb.CockroachDBOptions; +import sqlancer.cockroachdb.CockroachDBProvider; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; + +public class TestCockroachDBQueryPlan { + + @Test + void testCockroachDBQueryPlan() throws Exception { + String cockroachDB = System.getenv("COCKROACHDB_AVAILABLE"); + boolean cockroachDBIsAvailable = cockroachDB != null && cockroachDB.equalsIgnoreCase("true"); + assumeTrue(cockroachDBIsAvailable); + + String databaseName = "cockroachdb"; + CockroachDBProvider provider = new CockroachDBProvider(); + CockroachDBGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + CockroachDBOptions cockroachdbOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(cockroachdbOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals("left-join (cross);scan t2;select;scan t1;filters;filters (true);", queryPlan); + } + +} diff --git a/test/sqlancer/qpg/sqlite/TestSQLiteQPG.java b/test/sqlancer/qpg/sqlite/TestSQLiteQPG.java new file mode 100644 index 000000000..488037237 --- /dev/null +++ b/test/sqlancer/qpg/sqlite/TestSQLiteQPG.java @@ -0,0 +1,23 @@ +package sqlancer.qpg.sqlite; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestSQLiteQPG { + + @Test + public void testSqliteQPG() { + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", + "pqsdb" /* Workaround for connections not being closed */, "--qpg-enable", "true", "sqlite3", + "--oracle", "NoREC", "--test-fts", "false", "--test-rtree", "false", "--test-check-constraints", + "false", "--test-in-operator", "false" })); + } + +} diff --git a/test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java b/test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java new file mode 100644 index 000000000..7672f7edf --- /dev/null +++ b/test/sqlancer/qpg/sqlite/TestSQLiteQueryPlan.java @@ -0,0 +1,41 @@ +package sqlancer.qpg.sqlite; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3Options; +import sqlancer.sqlite3.SQLite3Provider; + +public class TestSQLiteQueryPlan { + + @Test + void testSQLiteQueryPlan() throws Exception { + String databaseName = "sqlite"; + SQLite3Provider provider = new SQLite3Provider(); + SQLite3GlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + SQLite3Options sqlite3Option = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(sqlite3Option); + state.setDatabaseName(databaseName); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals("SCAN t1;SCAN t2;RIGHT-JOIN t2;SCAN t2;", queryPlan); + } + +} diff --git a/test/sqlancer/qpg/tidb/TestTiDBQPG.java b/test/sqlancer/qpg/tidb/TestTiDBQPG.java new file mode 100644 index 000000000..66a43a41a --- /dev/null +++ b/test/sqlancer/qpg/tidb/TestTiDBQPG.java @@ -0,0 +1,24 @@ +package sqlancer.qpg.tidb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestTiDBQPG { + + @Test + public void testTiDBQPG() { + String tiDB = System.getenv("TIDB_AVAILABLE"); + boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); + assumeTrue(tiDBIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "tidb", + "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java b/test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java new file mode 100644 index 000000000..d07bebe61 --- /dev/null +++ b/test/sqlancer/qpg/tidb/TestTiDBQueryPlan.java @@ -0,0 +1,49 @@ +package sqlancer.qpg.tidb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBOptions; +import sqlancer.tidb.TiDBProvider; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; + +public class TestTiDBQueryPlan { + + @Test + void testTiDBQueryPlan() throws Exception { + String tiDB = System.getenv("TIDB_AVAILABLE"); + boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); + assumeTrue(tiDBIsAvailable); + + String databaseName = "tidb"; + TiDBProvider provider = new TiDBProvider(); + TiDBGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + TiDBOptions TiDBOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(TiDBOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals( + "HashJoin_7;TableReader_10(Build);Selection_9;TableFullScan_8;TableReader_12(Probe);TableFullScan_11;", + queryPlan); + } + +} From 7bd657d52af8d04a530f8b8671efa968749603a0 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 3 Jan 2023 07:36:30 +0800 Subject: [PATCH 293/856] Update CockroachDB version in Action (#675) --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b810c925b..2cf55af4e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,11 +109,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.0-beta.4.linux-amd64.tgz | tar xvz - cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-alpha.1.linux-amd64.tgz | tar xvz + cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -132,11 +132,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.0-beta.4.linux-amd64.tgz | tar xvz - cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-alpha.1.linux-amd64.tgz | tar xvz + cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v22.2.0-beta.4.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test From 91c4f9baa8c11eaa0a3b305eaae3ea1646486b65 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Wed, 4 Jan 2023 19:59:08 +0800 Subject: [PATCH 294/856] tidb: support USE_INDEX_MERGE(TableName) Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBHintGenerator.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index 06105ed4c..b1a343c67 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -96,7 +96,15 @@ private void generate() { twoTablesHint("SHUFFLE_JOIN", table); break; case USE_INDEX_MERGE: - twoTablesHint("USE_INDEX_MERGE", table); + if (Randomly.getBoolean()) { + if (table.hasIndexes()) { + tablesHint("USE_INDEX_MERGE"); + } else { + throw new IgnoreMeException(); + } + } else { + twoTablesHint("USE_INDEX_MERGE", table); + } break; case NO_INDEX_MERGE: sb.append("NO_INDEX_MERGE()"); From 06677bcf2f967d9c299622968c34c07e504c711d Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 5 Jan 2023 00:24:26 +0800 Subject: [PATCH 295/856] tidb: ignore fast ananlyze error (#683) * tidb: ignore fast ananlyze error Signed-off-by: Weizhen Wang * tidb: ignore fast ananlyze error Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index d84b117c2..8bcea30fb 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -3,6 +3,7 @@ import java.sql.SQLException; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -13,6 +14,7 @@ private TiDBAnalyzeTableGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { + ExpectedErrors errors = new ExpectedErrors(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); StringBuilder sb = new StringBuilder("ANALYZE "); @@ -30,7 +32,8 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx sb.append(Randomly.getNotCachedInteger(1, 1024)); sb.append(" BUCKETS"); } - return new SQLQueryAdapter(sb.toString()); + errors.add("Fast analyze hasn't reached General Availability and only support analyze version 1 currently"); + return new SQLQueryAdapter(sb.toString(), errors); } } From 7f1efbd4ba672f387ea2034b07bcbaf3b23e69d2 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 4 Jan 2023 19:49:19 +0100 Subject: [PATCH 296/856] Rewrite ClickHouse query generation (#681) * Rewrite ClickHouse query generation * Minor fix that triggered false positive * rename * Fix formatting --- src/sqlancer/clickhouse/ClickHouseErrors.java | 10 ++ .../clickhouse/ast/ClickHouseAggregate.java | 11 +- .../gen/ClickHouseColumnBuilder.java | 11 +- .../gen/ClickHouseExpressionGenerator.java | 144 +++++++++++++----- .../gen/ClickHouseTableGenerator.java | 12 +- .../oracle/norec/ClickHouseNoRECOracle.java | 79 +++++----- .../tlp/ClickHouseTLPAggregateOracle.java | 59 +++---- .../oracle/tlp/ClickHouseTLPBase.java | 21 +-- .../tlp/ClickHouseTLPGroupByOracle.java | 9 +- .../oracle/tlp/ClickHouseTLPHavingOracle.java | 40 ++--- .../oracle/tlp/ClickHouseTLPWhereOracle.java | 9 +- .../ast/ClickHouseToStringVisitorTest.java | 3 +- test/sqlancer/dbms/TestClickHouse.java | 97 ++++++++++-- 13 files changed, 323 insertions(+), 182 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index d2b6c700a..0548a04e9 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -62,7 +62,17 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("ILLEGAL_DIVISION"); errors.add("DECIMAL_OVERFLOW"); errors.add("Cannot convert out of range floating point value to integer type"); + errors.add("Unexpected inf or nan to integer conversion"); + errors.add("No such name in Block::erase"); // https://github.com/ClickHouse/ClickHouse/issues/42769 errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/42768 + errors.add("cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)"); + errors.add("AMBIGUOUS_IDENTIFIER"); + errors.add("CYCLIC_ALIASES"); + errors.add("Positional argument numeric constant expression is not representable as"); + errors.add("Positional argument must be constant with numeric type"); + errors.add(" is out of bounds. Expected in range"); + errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); + errors.add("Different order of columns in UNION subquery"); // https://github.com/ClickHouse/ClickHouse/issues/44866 } } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java index e868f82ca..a0483bd57 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseAggregate.java @@ -12,13 +12,12 @@ public class ClickHouseAggregate extends ClickHouseExpression { private final ClickHouseAggregate.ClickHouseAggregateFunction func; - private final List expr; + private final ClickHouseExpression expr; public enum ClickHouseAggregateFunction { AVG(ClickHouseDataType.Int8, ClickHouseDataType.Int16, ClickHouseDataType.Int32, ClickHouseDataType.Int64, ClickHouseDataType.UInt8, ClickHouseDataType.UInt16, ClickHouseDataType.UInt32, ClickHouseDataType.UInt64, ClickHouseDataType.Float32, ClickHouseDataType.Float64), - BOOL_AND(ClickHouseDataType.UInt8), BOOL_OR(ClickHouseDataType.UInt8), COUNT(ClickHouseDataType.Int8, ClickHouseDataType.Int16, ClickHouseDataType.Int32, ClickHouseDataType.Int64, ClickHouseDataType.UInt8, ClickHouseDataType.UInt16, ClickHouseDataType.UInt32, ClickHouseDataType.UInt64, ClickHouseDataType.Float32, ClickHouseDataType.Float64, @@ -42,8 +41,8 @@ public static ClickHouseAggregateFunction getRandom(ClickHouseDataType type) { return Randomly.fromOptions(values()); } - public List getTypes(ClickHouseDataType returnType) { - return Arrays.asList(returnType); + public ClickHouseDataType getType(ClickHouseDataType returnType) { + return returnType; } public boolean supportsReturnType(ClickHouseDataType returnType) { @@ -66,7 +65,7 @@ public ClickHouseSchema.ClickHouseLancerDataType getRandomReturnType() { } - public ClickHouseAggregate(List expr, ClickHouseAggregateFunction func) { + public ClickHouseAggregate(ClickHouseExpression expr, ClickHouseAggregateFunction func) { this.expr = expr; this.func = func; } @@ -75,7 +74,7 @@ public ClickHouseAggregate.ClickHouseAggregateFunction getFunc() { return func; } - public List getExpr() { + public ClickHouseExpression getExpr() { return expr; } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java index 65eb65623..2584ff0b3 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseColumnBuilder.java @@ -61,12 +61,11 @@ public String createColumn(String columnName, ClickHouseProvider.ClickHouseGloba if (allowMaterialized) { sb.append(" MATERIALIZED ("); sb.append( - ClickHouseVisitor - .asString(new ClickHouseExpressionGenerator(globalState) - .setColumns( - columns.stream().filter(p -> !p.getName().contentEquals(columnName)) - .collect(Collectors.toList())) - .generateExpression(dataType))); + ClickHouseVisitor.asString( + new ClickHouseExpressionGenerator(globalState).generateExpressionWithColumns( + columns.stream().filter(p -> !p.getName().contentEquals(columnName)) + .map(p -> p.asColumnReference(null)).collect(Collectors.toList()), + 2))); sb.append(")"); } break; diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index cd8ad8ff3..03750fee0 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -53,48 +53,134 @@ private enum Expression { UNARY_POSTFIX } - public ClickHouseExpression generateExpressionWithColumns(List columns, int depth) { - if (columns.isEmpty() || depth > 0 && Randomly.getBoolean()) { + public ClickHouseExpression generateExpressionWithColumns(List columns, + int remainingDepth) { + if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { return generateConstant(null); } - if (depth >= 2 || Randomly.getBoolean()) { + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { return columns.get((int) Randomly.getNotCachedInteger(0, columns.size() - 1)); } ColumnLike expr = Randomly.fromOptions(ColumnLike.values()); switch (expr) { case UNARY_PREFIX: - return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, depth + 1), + return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, remainingDepth - 1), ClickHouseUnaryPrefixOperator.MINUS); case BINARY_ARITHMETIC: - return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, depth + 1), - generateExpressionWithColumns(columns, depth + 1), + return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); case UNARY_FUNCTION: - return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, depth + 1), + return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); case BINARY_FUNCTION: - return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, depth + 1), - generateExpressionWithColumns(columns, depth + 1), + return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); default: throw new AssertionError(expr); } } + public ClickHouseExpression generateAggregateExpressionWithColumns(List columns, + int remainingDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return new ClickHouseAggregate(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseAggregate.ClickHouseAggregateFunction.getRandom()); + } + if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(null); + } + + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + return columns.get((int) Randomly.getNotCachedInteger(0, columns.size() - 1)); + } + + ColumnLike expr = Randomly.fromOptions(ColumnLike.values()); + switch (expr) { + case UNARY_PREFIX: + return new ClickHouseUnaryPrefixOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseUnaryPrefixOperator.MINUS); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation(generateExpressionWithColumns(columns, remainingDepth - 1), + generateExpressionWithColumns(columns, remainingDepth - 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); + default: + throw new AssertionError(expr); + } + } + + public ClickHouseExpression generateExpressionWithExpression(List expression, + int remainingDepth) { + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + if (Randomly.getBoolean()) { + return expression.get((int) Randomly.getNotCachedInteger(0, expression.size() - 1)); + } else { + return generateConstant(null); + } + } + + Expression type = Randomly.fromOptions(Expression.values()); + switch (type) { + case UNARY_PREFIX: + return new ClickHouseUnaryPrefixOperation(generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new ClickHouseUnaryPostfixOperation(generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.getRandom(), false); + case BINARY_COMPARISON: + return new ClickHouseBinaryComparisonOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryComparisonOperation.ClickHouseBinaryComparisonOperator.getRandomOperator()); + case BINARY_LOGICAL: + return new ClickHouseBinaryLogicalOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryLogicalOperation.ClickHouseBinaryLogicalOperator.getRandom()); + case BINARY_ARITHMETIC: + return new ClickHouseBinaryArithmeticOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryArithmeticOperation.ClickHouseBinaryArithmeticOperator.getRandom()); + case UNARY_FUNCTION: + return new ClickHouseUnaryFunctionOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseUnaryFunctionOperation.ClickHouseUnaryFunctionOperator.getRandom()); + case BINARY_FUNCTION: + return new ClickHouseBinaryFunctionOperation( + generateExpressionWithExpression(expression, remainingDepth - 1), + generateExpressionWithExpression(expression, remainingDepth - 1), + ClickHouseBinaryFunctionOperation.ClickHouseBinaryFunctionOperator.getRandom()); + default: + throw new AssertionError(type); + } + } + @Override protected ClickHouseExpression generateExpression(ClickHouseLancerDataType type, int depth) { - if (allowAggregateFunctions && Randomly.getBoolean()) { - return generateAggregate(); + if (allowAggregateFunctions && Randomly.getBooleanWithRatherLowProbability()) { + ClickHouseLancerDataType aggType = ClickHouseLancerDataType.getRandom(); + return new ClickHouseAggregate(generateExpression(aggType, depth + 1), + ClickHouseAggregate.ClickHouseAggregateFunction.getRandom()); } - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { return generateLeafNode(type); } Expression expr = Randomly.fromOptions(Expression.values()); ClickHouseLancerDataType leftLeafType = ClickHouseLancerDataType.getRandom(); ClickHouseLancerDataType rightLeafType = ClickHouseLancerDataType.getRandom(); - if (Randomly.getBoolean()) { + if (Randomly.getBooleanWithRatherLowProbability()) { rightLeafType = leftLeafType; } @@ -133,8 +219,8 @@ protected ClickHouseExpression.ClickHouseJoinOnClause generateJoinClause(ClickHo ClickHouseTableReference rightTable) { List leftColumns = leftTable.getColumnReferences(); List rightColumns = rightTable.getColumnReferences(); - ClickHouseExpression leftExpr = generateExpressionWithColumns(leftColumns, 0); - ClickHouseExpression rightExpr = generateExpressionWithColumns(rightColumns, 0); + ClickHouseExpression leftExpr = generateExpressionWithColumns(leftColumns, 2); + ClickHouseExpression rightExpr = generateExpressionWithColumns(rightColumns, 2); return new ClickHouseExpression.ClickHouseJoinOnClause(leftExpr, rightExpr); } @@ -223,20 +309,17 @@ public ClickHouseExpression generateConstant(ClickHouseLancerDataType genType) { } public ClickHouseExpression getHavingClause() { - allowAggregateFunctions = true; - return generateExpression(new ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + return generateAggregate(); } public ClickHouseAggregate generateArgsForAggregate(ClickHouseDataType dataType, ClickHouseAggregate.ClickHouseAggregateFunction agg) { - List types = agg.getTypes(dataType); - List args = new ArrayList<>(); - for (ClickHouseDataType argType : types) { - this.allowAggregateFunctions = false; - args.add(generateExpression(new ClickHouseLancerDataType(argType))); - this.allowAggregateFunctions = true; - } - return new ClickHouseAggregate(args, agg); + ClickHouseDataType type = agg.getType(dataType); + this.allowAggregateFunctions = false; + ClickHouseExpression arg = generateExpression(new ClickHouseLancerDataType(type)); + this.allowAggregateFunctions = true; + + return new ClickHouseAggregate(arg, agg); } public ClickHouseExpressionGenerator allowAggregates(boolean value) { @@ -245,19 +328,12 @@ public ClickHouseExpressionGenerator allowAggregates(boolean value) { } public ClickHouseExpression generateAggregate() { - return getAggregate(ClickHouseLancerDataType.getRandom().getType()); - } - - private ClickHouseExpression getAggregate(ClickHouseDataType dataType) { - List aggregates = ClickHouseAggregate.ClickHouseAggregateFunction - .getAggregates(dataType); - ClickHouseAggregate.ClickHouseAggregateFunction agg = Randomly.fromList(aggregates); - return generateArgsForAggregate(dataType, agg); + return generateAggregateExpressionWithColumns(columnRefs, 3); } @Override public ClickHouseExpression generatePredicate() { - return generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + return generateExpressionWithColumns(columnRefs, 3); } @Override diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index f3e5baecc..19ce78d27 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -4,8 +4,6 @@ import java.util.List; import java.util.stream.Collectors; -import com.clickhouse.client.ClickHouseDataType; - import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; @@ -82,7 +80,7 @@ public void start() { if (Randomly.getBoolean()) { sb.append(" ORDER BY "); ClickHouseExpression expr = gen.generateExpressionWithColumns( - columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); sb.append(ClickHouseToStringVisitor.asString(expr)); } else { sb.append(" ORDER BY tuple() "); @@ -91,13 +89,13 @@ public void start() { if (Randomly.getBoolean()) { sb.append(" PARTITION BY "); ClickHouseExpression expr = gen.generateExpressionWithColumns( - columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); sb.append(ClickHouseToStringVisitor.asString(expr)); } if (Randomly.getBoolean()) { sb.append(" SAMPLE BY "); ClickHouseExpression expr = gen.generateExpressionWithColumns( - columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 0); + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); sb.append(ClickHouseToStringVisitor.asString(expr)); } // TODO: PRIMARY KEY @@ -111,8 +109,8 @@ private void addColumnsConstraint(ClickHouseExpressionGenerator gen) { sb.append(" CONSTRAINT "); sb.append(ClickHouseCommon.createConstraintName(i)); sb.append(" CHECK "); - ClickHouseExpression expr = gen - .generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); + ClickHouseExpression expr = gen.generateExpressionWithColumns( + columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 2); sb.append(ClickHouseToStringVisitor.asString(expr)); } } diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java index cda529cbe..9818d198b 100644 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -1,24 +1,20 @@ package sqlancer.clickhouse.oracle.norec; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.SQLConnection; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; +import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; import sqlancer.clickhouse.ast.ClickHouseTableReference; @@ -31,27 +27,33 @@ public class ClickHouseNoRECOracle extends NoRECBase implements TestOracle { - private final ClickHouseSchema s; + private final ClickHouseSchema schema; public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { super(globalState); - this.s = globalState.getSchema(); + this.schema = globalState.getSchema(); ClickHouseErrors.addExpectedExpressionErrors(errors); } @Override public void check() throws SQLException { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state); + List tables = schema.getRandomTableNonEmptyTables().getTables(); + ClickHouseTableReference table = new ClickHouseTableReference( + tables.get((int) Randomly.getNotCachedInteger(0, tables.size() - 1)), "left"); + List columns = table.getColumnReferences(); - ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); - ClickHouseTable leftTable = randomTables.getTables().remove(0); - ClickHouseTableReference table = new ClickHouseTableReference(leftTable, "left"); - List columns = randomTables.getColumns(); - columns.addAll(leftTable.getColumns()); - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); - ClickHouseExpression randomWhereCondition = gen.generateExpression(ClickHouseLancerDataType.getRandom()); - List joins = gen.getRandomJoinClauses(table, randomTables.getTables()); - int secondCount = getSecondQuery(table, randomWhereCondition, joins); - int firstCount = getFirstQueryCount(con, table, columns, randomWhereCondition, joins); + List joinStatements = new ArrayList<>(); + if (state.getClickHouseOptions().testJoins && Randomly.getBoolean()) { + joinStatements = gen.getRandomJoinClauses(table, tables); + columns.addAll(joinStatements.stream().flatMap(j -> j.getRightTable().getColumnReferences().stream()) + .collect(Collectors.toList())); + } + gen.addColumns(columns); + + ClickHouseExpression randomWhereCondition = gen.generateExpressionWithColumns(columns, 5); + int secondCount = getSecondQuery(table, randomWhereCondition, joinStatements); + int firstCount = getFirstQueryCount(table, columns, randomWhereCondition, joinStatements); if (firstCount == -1 || secondCount == -1) { throw new IgnoreMeException(); } @@ -91,34 +93,35 @@ private int getSecondQuery(ClickHouseExpression table, ClickHouseExpression wher return secondCount; } - private int getFirstQueryCount(SQLConnection con, ClickHouseExpression tableList, List columns, + private int getFirstQueryCount(ClickHouseExpression tableList, List columns, ClickHouseExpression randomWhereCondition, List joins) throws SQLException { ClickHouseSelect select = new ClickHouseSelect(); - List allColumns = columns.stream().map(c -> c.asColumnReference(null)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); + List filteredColumns = Randomly.extractNrRandomColumns(columns, + (int) Randomly.getNotCachedInteger(1, columns.size())); + select.setFetchColumns( + filteredColumns.stream().map(c -> (ClickHouseExpression) c).collect(Collectors.toList())); select.setFromClause(tableList); select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( - new ClickHouseExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } select.setJoinClauses(joins); int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = ClickHouseToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); + optimizedQueryString = ClickHouseToStringVisitor.asString(select); + SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); + SQLancerResultSet rs; + + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(optimizedQueryString, e); + } + if (rs == null) { + return -1; } + + while (rs.next()) { + firstCount++; + } + rs.close(); return firstCount; } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index 2646878a2..d14356a57 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -3,23 +3,16 @@ import java.sql.SQLException; import java.util.Arrays; import java.util.List; - -import com.clickhouse.client.ClickHouseDataType; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseAliasOperation; -import sqlancer.clickhouse.ast.ClickHouseExpression; -import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.ast.ClickHouseTableReference; -import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; -import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; public class ClickHouseTLPAggregateOracle extends ClickHouseTLPBase { @@ -30,55 +23,41 @@ public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState sta @Override public void check() throws SQLException { - ClickHouseSchema s = state.getSchema(); - ClickHouseSchema.ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); - ClickHouseSchema.ClickHouseTable table = randomTables.getTables().remove(0); - ClickHouseTableReference tableRef = new ClickHouseTableReference(table, table.getName()); - List columns = randomTables.getColumns(); - columns.addAll(table.getColumns()); - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); - List joins = gen.getRandomJoinClauses(tableRef, randomTables.getTables()); + super.check(); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); + } - gen = new ClickHouseExpressionGenerator(state).setColumns(randomTables.getColumns()); - ClickHouseSelect select = new ClickHouseSelect(); ClickHouseAggregate.ClickHouseAggregateFunction windowFunction = Randomly.fromOptions( ClickHouseAggregate.ClickHouseAggregateFunction.MIN, ClickHouseAggregate.ClickHouseAggregateFunction.MAX, ClickHouseAggregate.ClickHouseAggregateFunction.SUM); - ClickHouseAggregate aggregate = new ClickHouseAggregate( - gen.generateExpressions(ClickHouseSchema.ClickHouseLancerDataType.getRandom(), 1), windowFunction); + + ClickHouseAggregate aggregate = new ClickHouseAggregate(gen.generateExpressionWithColumns(columns, 6), + windowFunction); select.setFetchColumns(Arrays.asList(aggregate)); - ClickHouseExpression from = tableRef; - select.setFromClause(from); - select.setJoinClauses(joins); - if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); - } + String originalQuery = ClickHouseVisitor.asString(select); originalQuery += " SETTINGS aggregate_functions_null_for_empty = 1"; - ClickHouseExpression whereClause = gen - .generateExpression(new ClickHouseSchema.ClickHouseLancerDataType(ClickHouseDataType.UInt8)); - ClickHouseUnaryPrefixOperation negatedClause = new ClickHouseUnaryPrefixOperation(whereClause, - ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator.NOT); - ClickHouseUnaryPostfixOperation notNullClause = new ClickHouseUnaryPostfixOperation(whereClause, - ClickHouseUnaryPostfixOperation.ClickHouseUnaryPostfixOperator.IS_NULL, false); - select.setFetchColumns(Arrays.asList(new ClickHouseAliasOperation(aggregate, "aggr"))); - select.setFromClause(from); - select.setWhereClause(whereClause); + + select.setWhereClause(predicate); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setGroupByClause(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setGroupByClause(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } String metamorphicText = "SELECT " + aggregate.getFunc().toString() + "(aggr) FROM ("; metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; - select.setWhereClause(negatedClause); + select.setWhereClause(negatedPredicate); metamorphicText += ClickHouseVisitor.asString(select) + " UNION ALL "; - select.setWhereClause(notNullClause); + select.setWhereClause(isNullPredicate); metamorphicText += ClickHouseVisitor.asString(select); metamorphicText += ")"; metamorphicText += " SETTINGS aggregate_functions_null_for_empty = 1"; diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java index 690994ec0..9374442ff 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPBase.java @@ -13,7 +13,6 @@ import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTables; import sqlancer.clickhouse.ClickHouseVisitor; import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; @@ -29,7 +28,7 @@ public class ClickHouseTLPBase extends TernaryLogicPartitioningOracleBase { ClickHouseSchema schema; - ClickHouseTables targetTables; + List columns; ClickHouseExpressionGenerator gen; ClickHouseSelect select; @@ -42,26 +41,30 @@ public ClickHouseTLPBase(ClickHouseGlobalState state) { public void check() throws SQLException { gen = new ClickHouseExpressionGenerator(state); schema = state.getSchema(); - initializeTernaryPredicateVariants(); select = new ClickHouseSelect(); List tables = schema.getRandomTableNonEmptyTables().getTables(); ClickHouseTableReference table = new ClickHouseTableReference( tables.get((int) Randomly.getNotCachedInteger(0, tables.size())), Randomly.getBoolean() ? "left" : null); select.setFromClause(table); - List columns = table.getColumnReferences(); + columns = table.getColumnReferences(); if (state.getClickHouseOptions().testJoins && Randomly.getBoolean()) { List joinStatements = gen.getRandomJoinClauses(table, tables); - select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); + columns.addAll(joinStatements.stream().flatMap(j -> j.getRightTable().getColumnReferences().stream()) + .collect(Collectors.toList())); + select.setJoinClauses(joinStatements); } gen.addColumns(columns); - range(0, Randomly.smallNumber()).mapToObj(i -> gen.generateExpressionWithColumns(columns, 0)) - .collect(Collectors.toList()); - select.setFetchColumns(generateFetchColumns(columns)); + int small = Randomly.smallNumber(); + List from = range(0, 1 + small) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList()); + select.setFetchColumns(from); select.setWhereClause(null); + initializeTernaryPredicateVariants(); // Smoke check - ComparatorHelper.getResultSetFirstColumnAsString(ClickHouseVisitor.asString(select), errors, state); + String query = ClickHouseVisitor.asString(select); + ComparatorHelper.getResultSetFirstColumnAsString(query, errors, state); } List generateFetchColumns(List columns) { diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index 8aff8fd77..25bfff57f 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -3,10 +3,14 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.ComparatorHelper; +import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ast.ClickHouseExpression; public class ClickHouseTLPGroupByOracle extends ClickHouseTLPBase { @@ -17,7 +21,10 @@ public ClickHouseTLPGroupByOracle(ClickHouseProvider.ClickHouseGlobalState state @Override public void check() throws SQLException { super.check(); - select.setGroupByClause(select.getFetchColumns()); + List groupByColumns = IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList()); + + select.setGroupByClause(groupByColumns); select.setWhereClause(null); String originalQueryString = ClickHouseVisitor.asString(select); diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java index d71e17624..a6f3e0cdb 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPHavingOracle.java @@ -11,15 +11,12 @@ import sqlancer.Randomly; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseVisitor; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; +import sqlancer.clickhouse.ast.ClickHouseAggregate; import sqlancer.clickhouse.ast.ClickHouseExpression; import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; -import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; public class ClickHouseTLPHavingOracle extends ClickHouseTLPBase { @@ -30,33 +27,15 @@ public ClickHouseTLPHavingOracle(ClickHouseProvider.ClickHouseGlobalState state) @Override public void check() throws SQLException { - ClickHouseSchema s = state.getSchema(); - ClickHouseSchema.ClickHouseTables randomTables = s.getRandomTableNonEmptyTables(); - ClickHouseSchema.ClickHouseTable table = randomTables.getTables().remove(0); - ClickHouseTableReference tableRef = new ClickHouseTableReference(table, table.getName()); - List columns = randomTables.getColumns(); - columns.addAll(table.getColumns()); - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state).setColumns(columns); - List joins = gen.getRandomJoinClauses(tableRef, randomTables.getTables()); - List colRefs = joins.stream() - .flatMap(j -> j.getRightTable().getColumnReferences().stream()).collect(Collectors.toList()); - - ClickHouseExpressionGenerator aggrGen = new ClickHouseExpressionGenerator(state).allowAggregates(true) - .setColumns(columns); - ClickHouseSelect select = new ClickHouseSelect(); - select.setFetchColumns(aggrGen.generateExpressions(Randomly.smallNumber() + 1)); - ClickHouseTableReference from = new ClickHouseTableReference(table, table.getName()); - select.setJoinClauses(joins); + super.check(); + select.setFetchColumns(IntStream.range(0, Randomly.smallNumber() + 1) + .mapToObj(i -> gen.generateAggregateExpressionWithColumns(columns, 3)).collect(Collectors.toList())); select.setSelectType(ClickHouseSelect.SelectType.ALL); - select.setFromClause(from); // TODO order by? - List groupByColumns = IntStream.range(0, Randomly.smallNumber()) - .mapToObj(i -> gen.generateExpressionWithColumns(colRefs, 0)).collect(Collectors.toList()); + List groupByColumns = IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 6)).collect(Collectors.toList()); - if (groupByColumns.isEmpty()) { - throw new IgnoreMeException(); - } select.setGroupByClause(groupByColumns); select.setHavingClause(null); String originalQueryString = ClickHouseVisitor.asString(select); @@ -64,7 +43,12 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - ClickHouseExpression predicate = aggrGen.getHavingClause(); + List aggregateExprs = select.getFetchColumns().stream() + .filter(p -> p instanceof ClickHouseAggregate).collect(Collectors.toList()); + if (aggregateExprs.isEmpty()) { + throw new IgnoreMeException(); + } + ClickHouseExpression predicate = gen.generateExpressionWithExpression(aggregateExprs, 6); select.setHavingClause(predicate); String firstQueryString = ClickHouseVisitor.asString(select); select.setHavingClause(new ClickHouseUnaryPrefixOperation(predicate, diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java index 1976ca957..92195bedf 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java @@ -3,6 +3,8 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.ComparatorHelper; import sqlancer.Randomly; @@ -21,15 +23,18 @@ public ClickHouseTLPWhereOracle(ClickHouseProvider.ClickHouseGlobalState state) public void check() throws SQLException { super.check(); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } String originalQueryString = ClickHouseVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } + select.setWhereClause(predicate); String firstQueryString = ClickHouseVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java index 7231aced8..ba9628037 100644 --- a/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java +++ b/test/sqlancer/clickhouse/ast/ClickHouseToStringVisitorTest.java @@ -164,8 +164,7 @@ void selectSumAGroupByBTest() { ClickHouseColumnReference a_ref = a_col.asColumnReference(null); ClickHouseColumnReference b_ref = b_col.asColumnReference(null); ClickHouseSelect selectAB = new ClickHouseSelect(); - ClickHouseAggregate sum_a = new ClickHouseAggregate(Arrays.asList(a_ref), - ClickHouseAggregate.ClickHouseAggregateFunction.SUM); + ClickHouseAggregate sum_a = new ClickHouseAggregate(a_ref, ClickHouseAggregate.ClickHouseAggregateFunction.SUM); selectAB.setFetchColumns(Arrays.asList(sum_a)); selectAB.setFromClause(table_ref); selectAB.setGroupByClause(Arrays.asList(b_ref)); diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index 073fdcbd3..ae861dfd9 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -16,8 +16,8 @@ public void testClickHouseTLPWhereGroupBy() { assumeTrue(clickHouseIsAvailable); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", - "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPWhere", - "--oracle", "TLPGroupBy")); + "5", "--username", "default", "--password", "", "--database-prefix", "T1_", "clickhouse", + "--oracle", "TLPWhere", "--oracle", "TLPGroupBy")); } @Test @@ -25,8 +25,10 @@ public void testClickHouseTLPWhere() { String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); assumeTrue(clickHouseIsAvailable); - assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, - "--num-threads", "5", "--username", "default", "--password", "", "clickhouse", "--oracle", "TLPWhere")); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T2_", "clickhouse", + "--oracle", "TLPWhere")); } @Test @@ -37,7 +39,7 @@ public void testClickHouseTLPHaving() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", - "--password", "", "clickhouse", "--oracle", "TLPHaving")); + "--password", "", "--database-prefix", "T3_", "clickhouse", "--oracle", "TLPHaving")); } @Test @@ -48,7 +50,7 @@ public void testClickHouseTLPGroupBy() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", - "--password", "", "clickhouse", "--oracle", "TLPGroupBy")); + "--password", "", "--database-prefix", "T4_", "clickhouse", "--oracle", "TLPGroupBy")); } @Test @@ -59,7 +61,7 @@ public void testClickHouseTLPDistinct() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", - "--password", "", "clickhouse", "--oracle", "TLPDistinct")); + "--password", "", "--database-prefix", "T5_", "clickhouse", "--oracle", "TLPDistinct")); } @Test @@ -70,7 +72,7 @@ public void testClickHouseTLPAggregate() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", - "--password", "", "clickhouse", "--oracle", "TLPAggregate")); + "--password", "", "--database-prefix", "T6_", "clickhouse", "--oracle", "TLPAggregate")); } @Test @@ -81,7 +83,84 @@ public void testClickHouseNoREC() { assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", - "--password", "", "clickhouse", "--oracle", "NoREC")); + "--password", "", "--database-prefix", "T7_", "clickhouse", "--oracle", "NoREC")); + } + + @Test + public void testClickHouseTLPWhereGroupByWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T8_", "clickhouse", + "--oracle", "TLPWhere", "--oracle", "TLPGroupBy")); + } + + @Test + public void testClickHouseTLPWhereWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", + "5", "--username", "default", "--password", "", "--database-prefix", "T9_", "clickhouse", + "--oracle", "TLPWhere")); + } + + @Test + public void testClickHouseTLPHavingWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "--database-prefix", "T10_", "clickhouse", "--oracle", "TLPHaving")); + } + + @Test + public void testClickHouseTLPGroupByWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T11_", "clickhouse", "--oracle", "TLPGroupBy")); + } + + @Test + public void testClickHouseTLPDistinctWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T12_", "clickhouse", "--oracle", "TLPDistinct")); + } + + @Test + public void testClickHouseTLPAggregateWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", + "--password", "", "--database-prefix", "T13_", "clickhouse", "--oracle", "TLPAggregate")); + } + + @Test + public void testClickHouseNoRECWithJoins() { + String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); + boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); + assumeTrue(clickHouseIsAvailable); + assertEquals(0, + Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", + "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", + "--password", "", "--database-prefix", "T14_", "clickhouse", "--oracle", "NoREC")); } } From 427d6a45f74973afd3f34d7044bff17a5c52524c Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Fri, 6 Jan 2023 23:11:13 +0100 Subject: [PATCH 297/856] Disable SEMI JOIN in ClickHouse (#685) * Disable SEMI JOIN * fix * disable NATURAL --- src/sqlancer/clickhouse/ClickHouseErrors.java | 1 + src/sqlancer/clickhouse/ClickHouseToStringVisitor.java | 6 ------ src/sqlancer/clickhouse/ast/ClickHouseExpression.java | 4 +++- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 0548a04e9..2e03b17fd 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -73,6 +73,7 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add(" is out of bounds. Expected in range"); errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); errors.add("Different order of columns in UNION subquery"); // https://github.com/ClickHouse/ClickHouse/issues/44866 + errors.add("Unexpected inf or nan to integer conversion"); } } diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index cbc622560..37f4d3ab6 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -151,12 +151,6 @@ public void visit(ClickHouseExpression.ClickHouseJoin join) { } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.FULL_OUTER) { sb.append(" FULL OUTER JOIN "); visit(join.getRightTable()); - } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_SEMI) { - sb.append(" LEFT SEMI JOIN "); - visit(join.getRightTable()); - } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.RIGHT_SEMI) { - sb.append(" RIGHT SEMI JOIN "); - visit(join.getRightTable()); } else if (type == ClickHouseExpression.ClickHouseJoin.JoinType.LEFT_ANTI) { sb.append(" LEFT ANTI JOIN "); visit(join.getRightTable()); diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index 34eda72a5..f5cd2ed1e 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -63,8 +63,10 @@ public String getOperatorRepresentation() { public static class ClickHouseJoin extends ClickHouseExpression { // TODO: support ANY, ALL, ASOF modifiers + // LEFT_SEMI, RIGHT_SEMI are not deterministic as ClickHouse allows to read columns from + // whitelist table as well public enum JoinType { - INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, RIGHT_SEMI, LEFT_ANTI, RIGHT_ANTI; + INNER, CROSS, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_ANTI, RIGHT_ANTI; } private final ClickHouseTableReference leftTable; From be4d62f8efce0525f627eb71ed5569dd9368f7fe Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 12 Jan 2023 15:10:18 +0100 Subject: [PATCH 298/856] Fix comparator for GroupBy Oracle in ClickHouse (#691) --- .../clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java index 25bfff57f..e8881cb84 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPGroupByOracle.java @@ -37,8 +37,8 @@ public void check() throws SQLException { select.setWhereClause(isNullPredicate); String thirdQueryString = ClickHouseVisitor.asString(select); List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, - secondQueryString, thirdQueryString, combinedString, false, state, errors); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); } From 834f646f428e4a937d5cf3d6b75b686bdec80bc7 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Fri, 13 Jan 2023 13:47:54 +0100 Subject: [PATCH 299/856] Update ClickHouse expected errors (#687) * Update errors * Update ClickHouseErrors.java --- src/sqlancer/clickhouse/ClickHouseErrors.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 2e03b17fd..5a115ebd5 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -38,7 +38,6 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Sorting key cannot contain constants"); errors.add("There is no supertype for types"); errors.add("argument of function"); - errors.add("because it is constant but values of constants are different in source and result"); // https://github.com/ClickHouse/ClickHouse/issues/22119 errors.add("but its arguments considered equal according to constraints"); errors.add("does not return a value of type UInt8"); errors.add("does not return a value of type UInt8"); @@ -64,7 +63,7 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Cannot convert out of range floating point value to integer type"); errors.add("Unexpected inf or nan to integer conversion"); errors.add("No such name in Block::erase"); // https://github.com/ClickHouse/ClickHouse/issues/42769 - errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/42768 + errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/43003 errors.add("cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)"); errors.add("AMBIGUOUS_IDENTIFIER"); errors.add("CYCLIC_ALIASES"); @@ -74,6 +73,8 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); errors.add("Different order of columns in UNION subquery"); // https://github.com/ClickHouse/ClickHouse/issues/44866 errors.add("Unexpected inf or nan to integer conversion"); + errors.add("Unsigned type must not contain"); + errors.add("Unexpected inf or nan to integer conversion"); } } From 1c053cef10b3a1b9b66d8a31892ca8949073d068 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 15 Jan 2023 23:33:45 +0800 Subject: [PATCH 300/856] tidb: support tidb_cost_model_version (#692) Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 4 ++-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cf55af4e..c51c32270 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -328,8 +328,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + docker pull pingcap/tidb:nightly + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:nightly sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index b48ea31cc..222947807 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -13,7 +13,7 @@ private TiDBSetGenerator() { } private enum Action { - + TIDB_COST_MODEL_VERSION("tidb_cost_model_version", (r) -> Randomly.fromOptions(1, 2)), // // SQL_MODE("sql_mode", (r) -> Randomly.fromOptions("TRADITIONAL", "ANSI", "POSTGRESQL", "ORACLE")), TIDB_OPT_AGG_PUSH_DOWN("tidb_opt_agg_push_down", (r) -> Randomly.fromOptions(0, 1)), // TIDB_BUILD_STATS_CONCURRENCY("tidb_build_stats_concurrency", (r) -> Randomly.getNotCachedInteger(0, 500)), // From e7d91ad032492ca050b62b01db733a03a622f7f1 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 16 Jan 2023 09:42:56 +0800 Subject: [PATCH 301/856] Revert "tidb: support tidb_cost_model_version (#692)" (#694) This reverts commit 1c053cef10b3a1b9b66d8a31892ca8949073d068. --- .github/workflows/main.yml | 4 ++-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c51c32270..2cf55af4e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -328,8 +328,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:nightly - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:nightly + docker pull pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 222947807..b48ea31cc 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -13,7 +13,7 @@ private TiDBSetGenerator() { } private enum Action { - TIDB_COST_MODEL_VERSION("tidb_cost_model_version", (r) -> Randomly.fromOptions(1, 2)), // + // SQL_MODE("sql_mode", (r) -> Randomly.fromOptions("TRADITIONAL", "ANSI", "POSTGRESQL", "ORACLE")), TIDB_OPT_AGG_PUSH_DOWN("tidb_opt_agg_push_down", (r) -> Randomly.fromOptions(0, 1)), // TIDB_BUILD_STATS_CONCURRENCY("tidb_build_stats_concurrency", (r) -> Randomly.getNotCachedInteger(0, 500)), // From ea7de8bf76c0a6b3fc50659b6ed2b9e3f2f62e20 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 16 Jan 2023 22:52:13 +0800 Subject: [PATCH 302/856] tidb: not use DEFAULT if columns have not default value (#693) * tidb: not use DEFAULT if columns have not default value Signed-off-by: Weizhen Wang * tidb: not use DEFAULT if columns have not default value Signed-off-by: Weizhen Wang * tidb: not use DEFAULT if columns have not default value Signed-off-by: Weizhen Wang * tidb: not use DEFAULT if columns have not default value Signed-off-by: Weizhen Wang Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBExpressionGenerator.java | 6 +++++- src/sqlancer/tidb/TiDBSchema.java | 13 +++++++++++-- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 2 +- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index a8fb75da0..a1db6fe2a 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -66,7 +66,11 @@ protected TiDBExpression generateExpression(int depth) { if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } - return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(generateColumn())); + TiDBColumn column = Randomly.fromList(columns); + if (column.hasDefault()) { + return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(new TiDBColumnReference(column))); + } + throw new IgnoreMeException(); case UNARY_POSTFIX: return new TiDBUnaryPostfixOperation(generateExpression(depth + 1), TiDBUnaryPostfixOperator.getRandom()); case UNARY_PREFIX: diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 676475d4a..04bde5e33 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -160,11 +160,14 @@ public static class TiDBColumn extends AbstractTableColumn { @@ -343,7 +350,9 @@ private static List getTableColumns(SQLConnection con, String tableN String dataType = rs.getString("Type"); boolean isNullable = rs.getString("Null").contentEquals("YES"); boolean isPrimaryKey = rs.getString("Key").contains("PRI"); - TiDBColumn c = new TiDBColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable); + boolean hasDefault = rs.getString("Default") != null; + TiDBColumn c = new TiDBColumn(columnName, getColumnType(dataType), isPrimaryKey, isNullable, + hasDefault); columns.add(c); } } diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 0ce8190e7..4349b69ca 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -38,7 +38,7 @@ public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException allowPrimaryKey = Randomly.getBoolean(); primaryKeyAsTableConstraints = allowPrimaryKey && Randomly.getBoolean(); for (int i = 0; i < nrColumns; i++) { - TiDBColumn fakeColumn = new TiDBColumn("c" + i, null, false, false); + TiDBColumn fakeColumn = new TiDBColumn("c" + i, null, false, false, false); columns.add(fakeColumn); } TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(columns); From 67751dc3d83b880cd8c7ed53416b8b157a2ba2ea Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 18 Jan 2023 15:21:46 +0100 Subject: [PATCH 303/856] Ignore AMBIGUOUS_IDENTIFIER (#696) * Ignore AMBIGUOUS_IDENTIFIER * Ignore MULTIPLE_EXPRESSIONS_FOR_ALIAS --- src/sqlancer/clickhouse/ClickHouseErrors.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 5a115ebd5..385ae2e27 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -75,6 +75,11 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Unexpected inf or nan to integer conversion"); errors.add("Unsigned type must not contain"); errors.add("Unexpected inf or nan to integer conversion"); + + // The way we generate JOINs we can have ambiguous left table column without alias + // We may not count it as an issue, but it makes no sense to add more complex AST generation logic + errors.add("MULTIPLE_EXPRESSIONS_FOR_ALIAS"); + errors.add("AMBIGUOUS_IDENTIFIER"); // https://github.com/ClickHouse/ClickHouse/issues/45389 } } From 9275f1ddd1d3bb33c1a10f07f41ecf9b552fdfbd Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Thu, 26 Jan 2023 09:51:42 +0800 Subject: [PATCH 304/856] update cockroachdb in CI (#703) --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2cf55af4e..5988c56fd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,11 +109,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-alpha.1.linux-amd64.tgz | tar xvz - cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.3.linux-amd64.tgz | tar xvz + cd cockroach-v22.2.3.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -132,11 +132,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-alpha.1.linux-amd64.tgz | tar xvz - cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.3.linux-amd64.tgz | tar xvz + cd cockroach-v22.2.3.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.1.0-alpha.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test From 39c48cde9ff0ab29b950a8d3e35930b0949f36a0 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Sat, 28 Jan 2023 03:02:01 +0100 Subject: [PATCH 305/856] Use master build of ClickHouse and update expected errors (#702) * Remove skipped error * Use master ClickHouse build in tests * Fix one more time * Get fatal logs if ClickHouse crashed * Fix tab from vi * fix not found errors * Increase socket timeout --------- Co-authored-by: Manuel Rigger --- .github/workflows/main.yml | 11 +++++++++-- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 -- src/sqlancer/clickhouse/ClickHouseProvider.java | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5988c56fd..1587aee78 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -72,6 +72,7 @@ jobs: - name: Run Tests run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test + clickhouse: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest @@ -87,11 +88,17 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up ClickHouse run: | - docker pull clickhouse/clickhouse-server:latest - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:latest + docker pull clickhouse/clickhouse-server:head + docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:head sleep 5 - name: Run Tests run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test + - name: Show fatal errors + run: docker exec clickhouse-server grep Fatal /var/log/clickhouse-server/clickhouse-server.log || echo No Fatal Errors found + - name: Teardown ClickHouse server + run: | + docker stop clickhouse-server + docker rm clickhouse-server cockroachdb: diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 385ae2e27..97a5e300c 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -40,7 +40,6 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("argument of function"); errors.add("but its arguments considered equal according to constraints"); errors.add("does not return a value of type UInt8"); - errors.add("does not return a value of type UInt8"); errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 errors.add("invalid character class range"); @@ -71,7 +70,6 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Positional argument must be constant with numeric type"); errors.add(" is out of bounds. Expected in range"); errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); - errors.add("Different order of columns in UNION subquery"); // https://github.com/ClickHouse/ClickHouse/issues/44866 errors.add("Unexpected inf or nan to integer conversion"); errors.add("Unsigned type must not contain"); errors.add("Unexpected inf or nan to integer conversion"); diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index e3a1fb21c..5c0c371c6 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -142,7 +142,8 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ e.printStackTrace(); } con.close(); - con = DriverManager.getConnection(String.format("jdbc:clickhouse://%s:%d/%s", host, port, databaseName), + con = DriverManager.getConnection( + String.format("jdbc:clickhouse://%s:%d/%s?socket_timeout=300000", host, port, databaseName), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); return new SQLConnection(con); } From 354d591cfcd37fa1de85ec77ec933d5d975e947a Mon Sep 17 00:00:00 2001 From: ZuoTiJia <49109132+ZuoTiJia@users.noreply.github.com> Date: Sat, 28 Jan 2023 20:59:50 +0800 Subject: [PATCH 306/856] Support CnosDB (#697) * Add Cnosdb (#1) Add Cnosdb * cnosdb: fix check style error * cnosdb: fix compiler error * cnosdb: remove unused * cnosdb: remove some pqs impl * cnosdb: remove dead code * add query execute count * refactor expectedError * cnosdb: remove unsuccessful count --- pom.xml | 5 + src/check_names.py | 1 + src/sqlancer/cnosdb/CnosDBBugs.java | 13 + .../cnosdb/CnosDBComparatorHelper.java | 145 +++++ .../cnosdb/CnosDBCompoundDataType.java | 20 + src/sqlancer/cnosdb/CnosDBExpectedError.java | 80 +++ src/sqlancer/cnosdb/CnosDBGlobalState.java | 28 + .../cnosdb/CnosDBLoggableFactory.java | 55 ++ src/sqlancer/cnosdb/CnosDBOptions.java | 64 +++ src/sqlancer/cnosdb/CnosDBProvider.java | 123 +++++ src/sqlancer/cnosdb/CnosDBSchema.java | 243 ++++++++ .../cnosdb/CnosDBToStringVisitor.java | 278 ++++++++++ src/sqlancer/cnosdb/CnosDBVisitor.java | 102 ++++ src/sqlancer/cnosdb/ast/CnosDBAggregate.java | 113 ++++ src/sqlancer/cnosdb/ast/CnosDBAlias.java | 35 ++ .../cnosdb/ast/CnosDBBetweenOperation.java | 34 ++ .../ast/CnosDBBinaryArithmeticOperation.java | 69 +++ .../ast/CnosDBBinaryComparisonOperation.java | 57 ++ .../ast/CnosDBBinaryLogicalOperation.java | 33 ++ .../cnosdb/ast/CnosDBCastOperation.java | 60 ++ .../cnosdb/ast/CnosDBColumnValue.java | 27 + .../cnosdb/ast/CnosDBConcatOperation.java | 22 + src/sqlancer/cnosdb/ast/CnosDBConstant.java | 520 ++++++++++++++++++ src/sqlancer/cnosdb/ast/CnosDBExpression.java | 14 + src/sqlancer/cnosdb/ast/CnosDBFunction.java | 30 + .../ast/CnosDBFunctionWithUnknownResult.java | 104 ++++ .../cnosdb/ast/CnosDBInOperation.java | 35 ++ src/sqlancer/cnosdb/ast/CnosDBJoin.java | 46 ++ .../cnosdb/ast/CnosDBLikeOperation.java | 22 + .../cnosdb/ast/CnosDBOrderByTerm.java | 37 ++ .../cnosdb/ast/CnosDBPostfixOperation.java | 97 ++++ .../cnosdb/ast/CnosDBPostfixText.java | 29 + .../cnosdb/ast/CnosDBPrefixOperation.java | 73 +++ src/sqlancer/cnosdb/ast/CnosDBSelect.java | 102 ++++ src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java | 28 + src/sqlancer/cnosdb/client/CnosDBClient.java | 110 ++++ .../cnosdb/client/CnosDBConnection.java | 27 + .../cnosdb/client/CnosDBException.java | 9 + .../cnosdb/client/CnosDBResultSet.java | 52 ++ src/sqlancer/cnosdb/gen/CnosDBCommon.java | 31 ++ .../cnosdb/gen/CnosDBExpressionGenerator.java | 461 ++++++++++++++++ .../cnosdb/gen/CnosDBInsertGenerator.java | 59 ++ .../cnosdb/gen/CnosDBTableGenerator.java | 77 +++ .../cnosdb/oracle/CnosDBNoRECBase.java | 23 + .../cnosdb/oracle/CnosDBNoRECOracle.java | 171 ++++++ .../oracle/tlp/CnosDBTLPAggregateOracle.java | 176 ++++++ .../cnosdb/oracle/tlp/CnosDBTLPBase.java | 112 ++++ .../oracle/tlp/CnosDBTLPHavingOracle.java | 65 +++ .../oracle/tlp/CnosDBTLPWhereOracle.java | 46 ++ .../cnosdb/query/CnosDBOtherQuery.java | 30 + .../cnosdb/query/CnosDBQueryAdapter.java | 41 ++ .../cnosdb/query/CnosDBQueryProvider.java | 6 + .../cnosdb/query/CnosDBSelectQuery.java | 38 ++ 53 files changed, 4278 insertions(+) create mode 100644 src/sqlancer/cnosdb/CnosDBBugs.java create mode 100644 src/sqlancer/cnosdb/CnosDBComparatorHelper.java create mode 100644 src/sqlancer/cnosdb/CnosDBCompoundDataType.java create mode 100644 src/sqlancer/cnosdb/CnosDBExpectedError.java create mode 100644 src/sqlancer/cnosdb/CnosDBGlobalState.java create mode 100644 src/sqlancer/cnosdb/CnosDBLoggableFactory.java create mode 100644 src/sqlancer/cnosdb/CnosDBOptions.java create mode 100644 src/sqlancer/cnosdb/CnosDBProvider.java create mode 100644 src/sqlancer/cnosdb/CnosDBSchema.java create mode 100644 src/sqlancer/cnosdb/CnosDBToStringVisitor.java create mode 100644 src/sqlancer/cnosdb/CnosDBVisitor.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBAggregate.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBAlias.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBCastOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBColumnValue.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBConstant.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBExpression.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunction.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBInOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBJoin.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixText.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBSelect.java create mode 100644 src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java create mode 100644 src/sqlancer/cnosdb/client/CnosDBClient.java create mode 100644 src/sqlancer/cnosdb/client/CnosDBConnection.java create mode 100644 src/sqlancer/cnosdb/client/CnosDBException.java create mode 100644 src/sqlancer/cnosdb/client/CnosDBResultSet.java create mode 100644 src/sqlancer/cnosdb/gen/CnosDBCommon.java create mode 100644 src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java create mode 100644 src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java create mode 100644 src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java create mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java create mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java create mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java create mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java create mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java create mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java create mode 100644 src/sqlancer/cnosdb/query/CnosDBOtherQuery.java create mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java create mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryProvider.java create mode 100644 src/sqlancer/cnosdb/query/CnosDBSelectQuery.java diff --git a/pom.xml b/pom.xml index b61126106..46211aac2 100644 --- a/pom.xml +++ b/pom.xml @@ -358,6 +358,11 @@ 2.7.1 runtime + + org.apache.commons + commons-csv + 1.9.0 + diff --git a/src/check_names.py b/src/check_names.py index bca448b19..72191ac7d 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -26,3 +26,4 @@ def verify_prefix(prefix, files): verify_prefix('Postgres', get_java_files("sqlancer/postgres/")) verify_prefix('SQLite3', get_java_files("sqlancer/sqlite3/")) verify_prefix('TiDB', get_java_files("sqlancer/tidb/")) +verify_prefix('CnosDB', get_java_files("sqlancer/cnosdb/")) diff --git a/src/sqlancer/cnosdb/CnosDBBugs.java b/src/sqlancer/cnosdb/CnosDBBugs.java new file mode 100644 index 000000000..4e6eb96e9 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBBugs.java @@ -0,0 +1,13 @@ +package sqlancer.cnosdb; + +public final class CnosDBBugs { + + // https://github.com/cnosdb/cnosdb/issues/786 + public static final boolean BUG786 = true; + + // https://github.com/apache/arrow-rs/issues/3547 + public static final boolean BUG3547 = true; + + private CnosDBBugs() { + } +} diff --git a/src/sqlancer/cnosdb/CnosDBComparatorHelper.java b/src/sqlancer/cnosdb/CnosDBComparatorHelper.java new file mode 100644 index 000000000..46b6ba615 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBComparatorHelper.java @@ -0,0 +1,145 @@ +package sqlancer.cnosdb; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.UnaryOperator; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.cnosdb.query.CnosDBSelectQuery; +import sqlancer.common.query.ExpectedErrors; + +public final class CnosDBComparatorHelper { + + private CnosDBComparatorHelper() { + } + + public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, + CnosDBGlobalState state) throws Exception { + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, errors); + List result = new ArrayList<>(); + CnosDBResultSet resultSet; + try { + q.executeAndGet(state); + resultSet = q.getResultSet(); + if (resultSet == null) { + throw new AssertionError(q); + } + while (resultSet.next()) { + result.add(resultSet.getString(1)); + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw e; + } + if (e instanceof NumberFormatException) { + throw new IgnoreMeException(); + } + if (e.getMessage() == null) { + throw new AssertionError(queryString, e); + } + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + throw new AssertionError(queryString, e); + } + + return result; + } + + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, + String originalQueryString, List combinedString, CnosDBGlobalState state) { + if (resultSet.size() != secondResultSet.size()) { + String queryFormatString = "-- %s;\n-- cardinality: %d"; + String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); + String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), + secondResultSet.size()); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", + resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); + throw new AssertionError(assertionMessage); + } + + Set firstHashSet = new HashSet<>(resultSet); + Set secondHashSet = new HashSet<>(secondResultSet); + + if (!firstHashSet.equals(secondHashSet)) { + Set firstResultSetMisses = new HashSet<>(firstHashSet); + firstResultSetMisses.removeAll(secondHashSet); + Set secondResultSetMisses = new HashSet<>(secondHashSet); + secondResultSetMisses.removeAll(firstHashSet); + String queryFormatString = "-- %s;\n-- misses: %s"; + String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); + String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), + secondResultSetMisses); + // update the SELECT queries to be logged at the bottom of the error log file + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", + firstQueryString, secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, + String originalQueryString, List combinedString, CnosDBGlobalState state, + UnaryOperator canonicalizationRule) { + // Overloaded version of assumeResultSetsAreEqual that takes a canonicalization function which is applied to + // both result sets before their comparison. + List canonicalizedResultSet = resultSet.stream().map(canonicalizationRule).collect(Collectors.toList()); + List canonicalizedSecondResultSet = secondResultSet.stream().map(canonicalizationRule) + .collect(Collectors.toList()); + assumeResultSetsAreEqual(canonicalizedResultSet, canonicalizedSecondResultSet, originalQueryString, + combinedString, state); + } + + public static List getCombinedResultSet(String firstQueryString, String secondQueryString, + String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, + ExpectedErrors errors) throws Exception { + List secondResultSet; + if (asUnion) { + String unionString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL " + + thirdQueryString; + combinedString.add(unionString); + secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); + } else { + secondResultSet = new ArrayList<>(); + secondResultSet.addAll(getResultSetFirstColumnAsString(firstQueryString, errors, state)); + secondResultSet.addAll(getResultSetFirstColumnAsString(secondQueryString, errors, state)); + secondResultSet.addAll(getResultSetFirstColumnAsString(thirdQueryString, errors, state)); + combinedString.add(firstQueryString); + combinedString.add(secondQueryString); + combinedString.add(thirdQueryString); + } + return secondResultSet; + } + + public static List getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString, + String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, + ExpectedErrors errors) throws Exception { + String unionString; + if (asUnion) { + unionString = firstQueryString + " UNION " + secondQueryString + " UNION " + thirdQueryString; + } else { + unionString = "SELECT DISTINCT * FROM (" + firstQueryString + " UNION ALL " + secondQueryString + + " UNION ALL " + thirdQueryString + ")"; + } + List secondResultSet; + combinedString.add(unionString); + secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); + return secondResultSet; + } +} diff --git a/src/sqlancer/cnosdb/CnosDBCompoundDataType.java b/src/sqlancer/cnosdb/CnosDBCompoundDataType.java new file mode 100644 index 000000000..034f0fc90 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBCompoundDataType.java @@ -0,0 +1,20 @@ +package sqlancer.cnosdb; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public final class CnosDBCompoundDataType { + + private final CnosDBDataType dataType; + + private CnosDBCompoundDataType(CnosDBDataType dataType) { + this.dataType = dataType; + } + + public static CnosDBCompoundDataType create(CnosDBDataType type) { + return new CnosDBCompoundDataType(type); + } + + public CnosDBDataType getDataType() { + return dataType; + } +} diff --git a/src/sqlancer/cnosdb/CnosDBExpectedError.java b/src/sqlancer/cnosdb/CnosDBExpectedError.java new file mode 100644 index 000000000..516e44ce4 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBExpectedError.java @@ -0,0 +1,80 @@ +package sqlancer.cnosdb; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class CnosDBExpectedError { + private static final List ERRORS = new ArrayList<>(); + + private CnosDBExpectedError() { + } + + static { + ERRORS.add("have the same name. Consider aliasing"); + ERRORS.add( + "error: Optimizer rule 'projection_push_down' failed due to unexpected error: Schema error: Schema contains duplicate qualified field name"); + ERRORS.add("Projection references non-aggregate values:"); + ERRORS.add("External err: Schema error: No field named"); + ERRORS.add( + "Optimizer rule 'common_sub_expression_eliminate' failed due to unexpected error: Schema error: No field named"); + ERRORS.add("Binary"); + ERRORS.add("Invalid pattern in LIKE expression"); + ERRORS.add("If the projection contains the time column, it must contain the field column."); + ERRORS.add("Schema error: No field named"); + ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:"); + ERRORS.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error"); + ERRORS.add("Schema error: No field named "); + ERRORS.add("err: External err: Schema error: No field named"); + ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error"); + ERRORS.add("Csv error: CSV Writer does not support List"); + ERRORS.add("This feature is not implemented: cross join."); + ERRORS.add("Execution error: field position must be greater than zero"); + ERRORS.add("First argument of `DATE_PART` must be non-null scalar Utf8"); + ERRORS.add("Cannot create filter with non-boolean predicate 'NULL' returning Null"); + ERRORS.add("requested character too large for encoding."); + ERRORS.add("Can not find compatible types to compare Boolean with [Utf8]."); + ERRORS.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT"); + ERRORS.add("HAVING clause references non-aggregate values:"); + ERRORS.add("Cannot create filter with non-boolean predicate"); + ERRORS.add("negative substring length not allowed"); + ERRORS.add("The function Sum does not support inputs of type Boolean."); + ERRORS.add("The function Avg does not support inputs of type Boolean."); + ERRORS.add("Percentile value must be between 0.0 and 1.0 inclusive"); + ERRORS.add("Date part '' not supported"); + ERRORS.add("Min/Max accumulator not implemented for type Boolean."); + ERRORS.add("meta need get_series_id_by_filter"); + ERRORS.add("Arrow: Cast error:"); + ERRORS.add("Arrow error: Cast error:"); + ERRORS.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:"); + ERRORS.add("Arrow error: Divide by zero error"); + ERRORS.add("desired percentile argument must be float literal"); + ERRORS.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)"); + ERRORS.add("Execution error: Date part"); + ERRORS.add("Physical plan does not support logical expression MIN(Boolean"); + ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64"); + ERRORS.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64."); + ERRORS.add("Data type UInt64 not supported for binary operation '#' on dyn arrays."); + ERRORS.add("Arrow: Divide by zero error"); + ERRORS.add("The function ApproxPercentileCont does not support inputs of type Null."); + ERRORS.add("can't be evaluated because there isn't a common type to coerce the types to"); + ERRORS.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug"); + ERRORS.add("The function ApproxMedian does not support inputs of type Null."); + ERRORS.add("null character not permitted."); + ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Null."); + ERRORS.add("This feature is not implemented"); + ERRORS.add("The function Avg does not support inputs of type Null."); + ERRORS.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]"); + ERRORS.add( + "Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed."); + ERRORS.add("Coercion from"); + } + + public static ExpectedErrors expectedErrors() { + ExpectedErrors res = new ExpectedErrors(); + res.addAll(ERRORS); + return res; + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBGlobalState.java b/src/sqlancer/cnosdb/CnosDBGlobalState.java new file mode 100644 index 000000000..9f34e03a5 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBGlobalState.java @@ -0,0 +1,28 @@ +package sqlancer.cnosdb; + +import sqlancer.ExecutionTimer; +import sqlancer.GlobalState; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.common.query.Query; + +public class CnosDBGlobalState extends GlobalState { + + @Override + protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { + boolean logExecutionTime = getOptions().logExecutionTime(); + if (success && getOptions().printSucceedingStatements()) { + System.out.println(q.getQueryString()); + } + if (logExecutionTime) { + getLogger().writeCurrent(" -- " + timer.end().asString()); + } + if (q.couldAffectSchema()) { + updateSchema(); + } + } + + @Override + public CnosDBSchema readSchema() throws Exception { + return CnosDBSchema.fromConnection(getConnection()); + } +} diff --git a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java new file mode 100644 index 000000000..a3f361b14 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java @@ -0,0 +1,55 @@ +package sqlancer.cnosdb; + +import java.io.PrintWriter; +import java.io.StringWriter; + +import sqlancer.cnosdb.query.CnosDBOtherQuery; +import sqlancer.cnosdb.query.CnosDBQueryAdapter; +import sqlancer.common.log.Loggable; +import sqlancer.common.log.LoggableFactory; +import sqlancer.common.log.LoggedString; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; + +public class CnosDBLoggableFactory extends LoggableFactory { + + @Override + protected Loggable createLoggable(String input, String suffix) { + String completeString = input; + if (!input.endsWith(";")) { + completeString += ";"; + } + if (suffix != null && suffix.length() != 0) { + completeString += suffix; + } + return new LoggedString(completeString); + } + + @Override + public CnosDBQueryAdapter getQueryForStateToReproduce(String queryString) { + return new CnosDBOtherQuery(queryString, CnosDBExpectedError.expectedErrors()); + } + + @Override + public CnosDBQueryAdapter commentOutQuery(Query query) { + String queryString = query.getLogString(); + String newQueryString = "-- " + queryString; + ExpectedErrors errors = new ExpectedErrors(); + return new CnosDBOtherQuery(newQueryString, errors); + } + + @Override + protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { + String sb = "-- Time: " + time + "\n" + "-- Database: " + databaseName + "\n" + "-- Database version: " + + databaseVersion + "\n" + "-- seed value: " + seedValue + "\n"; + return new LoggedString(sb); + } + + @Override + public Loggable convertStacktraceToLoggable(Throwable throwable) { + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + throwable.printStackTrace(pw); + return new LoggedString("--" + sw.toString().replace("\n", "\n--")); + } +} diff --git a/src/sqlancer/cnosdb/CnosDBOptions.java b/src/sqlancer/cnosdb/CnosDBOptions.java new file mode 100644 index 000000000..9a6da00c4 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBOptions.java @@ -0,0 +1,64 @@ +package sqlancer.cnosdb; + +import java.util.ArrayList; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.cnosdb.CnosDBOptions.CnosDBOracleFactory; +import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPAggregateOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPHavingOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPWhereOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; + +@Parameters(separators = "=", commandDescription = "CnosDB (default port: " + CnosDBOptions.DEFAULT_PORT + + ", default host: " + CnosDBOptions.DEFAULT_HOST + ")") +public class CnosDBOptions implements DBMSSpecificOptions { + + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 31001; + + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for CnosDB") + public List oracle = List.of(CnosDBOracleFactory.QUERY_PARTITIONING); + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the CnosDB", arity = 1) + public String connectionURL = String.format("http://%s:%d", CnosDBOptions.DEFAULT_HOST, CnosDBOptions.DEFAULT_PORT); + + @Override + public List getTestOracleFactory() { + return oracle; + } + + public enum CnosDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + return new CnosDBNoRECOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + return new CnosDBTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + List> oracles = new ArrayList<>(); + oracles.add(new CnosDBTLPWhereOracle(globalState)); + oracles.add(new CnosDBTLPHavingOracle(globalState)); + oracles.add(new CnosDBTLPAggregateOracle(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + } + + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBProvider.java b/src/sqlancer/cnosdb/CnosDBProvider.java new file mode 100644 index 000000000..8b69c53b3 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBProvider.java @@ -0,0 +1,123 @@ +package sqlancer.cnosdb; + +import java.util.Objects; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.ProviderAdapter; +import sqlancer.Randomly; +import sqlancer.StatementExecutor; +import sqlancer.cnosdb.client.CnosDBClient; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.cnosdb.gen.CnosDBInsertGenerator; +import sqlancer.cnosdb.gen.CnosDBTableGenerator; +import sqlancer.cnosdb.query.CnosDBOtherQuery; +import sqlancer.cnosdb.query.CnosDBQueryProvider; +import sqlancer.common.log.LoggableFactory; + +@AutoService(DatabaseProvider.class) +public class CnosDBProvider extends ProviderAdapter { + + protected String username; + protected String password; + protected String host; + protected int port; + protected String databaseName; + + public CnosDBProvider() { + super(CnosDBGlobalState.class, CnosDBOptions.class); + } + + protected CnosDBProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + protected static int mapActions(CnosDBGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + if (Objects.requireNonNull(a) == Action.INSERT) { + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + } else { + throw new AssertionError(a); + } + return nrPerformed; + + } + + @Override + protected void checkViewsAreValid(CnosDBGlobalState globalState) { + } + + @Override + public void generateDatabase(CnosDBGlobalState globalState) throws Exception { + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + prepareTables(globalState); + + } + + @Override + public CnosDBConnection createDatabase(CnosDBGlobalState globalState) throws Exception { + + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + databaseName = globalState.getDatabaseName(); + CnosDBClient client = new CnosDBClient(host, port, username, password, databaseName); + CnosDBConnection connection = new CnosDBConnection(client); + client.execute("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + client.execute("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + + return connection; + } + + protected void createTables(CnosDBGlobalState globalState, int numTables) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + String tableName = String.format("m%d", globalState.getSchema().getDatabaseTables().size()); + CnosDBOtherQuery createTable = CnosDBTableGenerator.generate(tableName); + globalState.executeStatement(createTable); + } + } + + protected void prepareTables(CnosDBGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + CnosDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public String getDBMSName() { + return "CnosDB".toLowerCase(); + } + + @Override + public LoggableFactory getLoggableFactory() { + return new CnosDBLoggableFactory(); + } + + public enum Action implements AbstractAction { + INSERT(CnosDBInsertGenerator::insert); + + private final CnosDBQueryProvider sqlQueryProvider; + + Action(CnosDBQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public CnosDBOtherQuery getQuery(CnosDBGlobalState state) throws Exception { + return new CnosDBOtherQuery(sqlQueryProvider.getQuery(state).getQueryString(), + CnosDBExpectedError.expectedErrors()); + } + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBSchema.java b/src/sqlancer/cnosdb/CnosDBSchema.java new file mode 100644 index 000000000..35d9d8887 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBSchema.java @@ -0,0 +1,243 @@ +package sqlancer.cnosdb; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import sqlancer.Randomly; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; + +public class CnosDBSchema extends AbstractSchema { + + private final String databaseName; + + public CnosDBSchema(List databaseTables, String databaseName) { + super(databaseTables); + this.databaseName = databaseName; + } + + public static CnosDBDataType getColumnType(String typeString) { + switch (typeString.toLowerCase()) { + case "bigint": + return CnosDBDataType.INT; + case "boolean": + return CnosDBDataType.BOOLEAN; + case "string": + return CnosDBDataType.STRING; + case "double": + return CnosDBDataType.DOUBLE; + case "bigint unsigned": + case "unsigned": + return CnosDBDataType.UINT; + case "timestamp": + return CnosDBDataType.TIMESTAMP; + default: + throw new AssertionError(typeString); + } + } + + public static CnosDBSchema fromConnection(CnosDBConnection con) throws Exception { + CnosDBResultSet tablesRes = con.getClient().executeQuery("SHOW TABLES"); + + List tables = new ArrayList<>(); + while (tablesRes.next()) { + String tableName = tablesRes.getString(1); + List columns = getTableColumns(con, tableName); + tables.add(new CnosDBTable(tableName, columns)); + } + + return new CnosDBSchema(tables, con.getClient().getDatabase()); + } + + protected static List getTableColumns(CnosDBConnection con, String tableName) throws Exception { + CnosDBResultSet columnsRes = con.getClient().executeQuery("DESCRIBE TABLE " + tableName); + List columns = new ArrayList<>(); + CnosDBTable table = new CnosDBTable(tableName, columns); + while (columnsRes.next()) { + String columnName = columnsRes.getString(1); + String columnType = columnsRes.getString(3).toLowerCase(); + CnosDBDataType dataType = CnosDBSchema.getColumnType(columnsRes.getString(2)); + CnosDBColumn column; + if (columnType.contentEquals("time")) { + column = new CnosDBTimeColumn(); + } else if (columnType.contentEquals("tag")) { + column = new CnosDBTagColumn(columnName); + } else { + column = new CnosDBFieldColumn(columnName, dataType); + } + column.setTable(table); + columns.add(column); + } + + return columns; + } + + public CnosDBTables getRandomTableNonEmptyTables() { + return new CnosDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public String getDatabaseName() { + return databaseName; + } + + public enum CnosDBDataType { + INT, BOOLEAN, STRING, DOUBLE, UINT, TIMESTAMP; + + public static CnosDBDataType getRandomType() { + return Randomly.fromOptions(values()); + } + + public static CnosDBDataType getRandomTypeWithoutTimeStamp() { + List dataTypes = new ArrayList<>(Arrays.asList(values())); + dataTypes.remove(TIMESTAMP); + return Randomly.fromList(dataTypes); + } + } + + public static class CnosDBColumn extends AbstractTableColumn { + + public CnosDBColumn(String name, CnosDBDataType columnType) { + super(name, null, columnType); + } + + public static CnosDBColumn createDummy(String name) { + return new CnosDBColumn(name, CnosDBDataType.INT); + } + + } + + public static class CnosDBTagColumn extends CnosDBColumn { + public CnosDBTagColumn(String name) { + super(name, CnosDBDataType.STRING); + } + } + + public static class CnosDBTimeColumn extends CnosDBColumn { + public CnosDBTimeColumn() { + super("TIME", CnosDBDataType.TIMESTAMP); + } + } + + public static class CnosDBFieldColumn extends CnosDBColumn { + public CnosDBFieldColumn(String name, CnosDBDataType columnType) { + super(name, columnType); + assert columnType != CnosDBDataType.TIMESTAMP; + } + } + + public static class CnosDBTables extends AbstractTables { + + public CnosDBTables(List tables) { + super(tables); + } + + public CnosDBRowValue getRandomRowValue(CnosDBConnection con) { + return null; + } + + public List getRandomColumnsWithOnlyOneField() { + ArrayList res = new ArrayList<>(); + this.getTables().forEach(table -> res.addAll(table.getRandomColumnsWithOnlyOneField())); + return res; + } + + } + + public static class CnosDBRowValue extends AbstractRowValue { + + protected CnosDBRowValue(CnosDBTables tables, Map values) { + super(tables, values); + } + + } + + public static class CnosDBTable extends AbstractTable { + + public CnosDBTable(String tableName, List columns) { + super(tableName, columns, null, false); + } + + @Override + public List getColumns() { + List res = super.getColumns(); + boolean hasTime = false; + for (CnosDBColumn column : res) { + if (column instanceof CnosDBTimeColumn) { + hasTime = true; + break; + } + } + assert hasTime; + + return res; + } + + public List getRandomColumnsWithOnlyOneField() { + ArrayList res = new ArrayList<>(); + boolean hasField = false; + for (CnosDBColumn column : getColumns()) { + if (column instanceof CnosDBTagColumn && Randomly.getBoolean()) { + res.add(column); + } else if (column instanceof CnosDBFieldColumn && !hasField) { + res.add(column); + hasField = true; + } + } + return res; + } + + // SELECT COUNT(*) FROM table; + @Override + public long getNrRows(CnosDBGlobalState globalState) { + long res; + try { + CnosDBResultSet tableCountRes = globalState.getConnection().getClient() + .executeQuery("SELECT COUNT(time) FROM " + this.name); + tableCountRes.next(); + res = tableCountRes.getLong(1); + } catch (Exception e) { + res = 0; + } + return res; + } + + @Override + public List getRandomNonEmptyColumnSubset() { + List selectedColumns = new ArrayList<>(); + ArrayList remainingColumns = new ArrayList<>(this.getColumns()); + + remainingColumns.removeIf(column -> column instanceof CnosDBTimeColumn); + CnosDBTimeColumn timeColumn = new CnosDBTimeColumn(); + timeColumn.setTable(this); + selectedColumns.add(timeColumn); + + remainingColumns.stream().filter(column -> column instanceof CnosDBTagColumn).findFirst().ifPresent(tag -> { + selectedColumns.add(tag); + remainingColumns.remove(tag); + }); + + remainingColumns.stream().filter(column -> column instanceof CnosDBFieldColumn).findFirst() + .ifPresent(field -> { + selectedColumns.add(field); + remainingColumns.remove(field); + }); + + int nr = Math.min(Randomly.smallNumber() + 1, remainingColumns.size()); + for (int i = 0; i < nr; i++) { + selectedColumns + .add(remainingColumns.remove((int) Randomly.getNotCachedInteger(0, remainingColumns.size()))); + } + return selectedColumns; + } + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java new file mode 100644 index 000000000..64f2fd8b8 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java @@ -0,0 +1,278 @@ +package sqlancer.cnosdb; + +import sqlancer.Randomly; +import sqlancer.cnosdb.ast.CnosDBAggregate; +import sqlancer.cnosdb.ast.CnosDBBetweenOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; +import sqlancer.cnosdb.ast.CnosDBCastOperation; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBFunction; +import sqlancer.cnosdb.ast.CnosDBInOperation; +import sqlancer.cnosdb.ast.CnosDBJoin; +import sqlancer.cnosdb.ast.CnosDBLikeOperation; +import sqlancer.cnosdb.ast.CnosDBOrderByTerm; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation; +import sqlancer.cnosdb.ast.CnosDBPostfixText; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; +import sqlancer.cnosdb.ast.CnosDBSimilarTo; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.ToStringVisitor; + +public final class CnosDBToStringVisitor extends ToStringVisitor implements CnosDBVisitor { + + @Override + public void visitSpecific(CnosDBExpression expr) { + CnosDBVisitor.super.visit(expr); + } + + @Override + public void visit(CnosDBConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(CnosDBPostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" "); + sb.append(op.getOperatorTextRepresentation()); + } + + @Override + public void visit(CnosDBColumnValue c) { + sb.append(c.getColumn().getFullQualifiedName()); + } + + @Override + public void visit(CnosDBPrefixOperation op) { + sb.append(op.getTextRepresentation()); + sb.append(" ("); + visit(op.getExpression()); + sb.append(")"); + } + + @Override + public void visit(CnosDBFromTable from) { + sb.append(from.getTable().getName()); + } + + @Override + public void visit(CnosDBSubquery subquery) { + sb.append("("); + visit(subquery.getSelect()); + sb.append(") AS "); + sb.append(subquery.getName()); + } + + @Override + public void visit(CnosDBSelect s) { + sb.append("SELECT "); + switch (s.getSelectOption()) { + case DISTINCT: + sb.append("DISTINCT "); + if (s.getDistinctOnClause() != null) { + sb.append("ON ("); + visit(s.getDistinctOnClause()); + sb.append(") "); + } + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + if (s.getFetchColumns() == null) { + sb.append("*"); + } else { + visit(s.getFetchColumns()); + } + sb.append(" FROM "); + visit(s.getFromList()); + + for (CnosDBJoin j : s.getJoinClauses()) { + sb.append(" "); + switch (j.getType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } + sb.append("JOIN"); + break; + case LEFT: + sb.append("LEFT OUTER JOIN"); + break; + case RIGHT: + sb.append("RIGHT OUTER JOIN"); + break; + case FULL: + sb.append("FULL OUTER JOIN"); + break; + // case CROSS: + // sb.append("CROSS JOIN"); + // break; + default: + throw new AssertionError(j.getType()); + } + sb.append(" "); + visit(j.getTableReference()); + // if (j.getType() != CnosDBJoinType.CROSS) { + sb.append(" ON "); + visit(j.getOnClause()); + // } + } + + if (s.getWhereClause() != null) { + sb.append(" WHERE "); + visit(s.getWhereClause()); + } + if (s.getGroupByExpressions().size() > 0) { + sb.append(" GROUP BY "); + visit(s.getGroupByExpressions()); + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + + } + if (!s.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(s.getOrderByExpressions()); + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(CnosDBOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder()); + } + + @Override + public void visit(CnosDBFunction f) { + sb.append(f.getFunctionName()); + sb.append("("); + int i = 0; + for (CnosDBExpression arg : f.getArguments()) { + if (i++ != 0) { + sb.append(", "); + } + visit(arg); + } + sb.append(")"); + } + + @Override + public void visit(CnosDBCastOperation cast) { + sb.append("CAST( "); + visit(cast.getExpression()); + sb.append(" AS "); + appendType(cast); + sb.append(")"); + } + + private void appendType(CnosDBCastOperation cast) { + CnosDBCompoundDataType compoundType = cast.getCompoundType(); + switch (compoundType.getDataType()) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: + sb.append("BIGINT"); + break; + case STRING: + sb.append(Randomly.fromOptions("STRING")); + break; + case DOUBLE: + sb.append("DOUBLE"); + break; + case UINT: + sb.append("BIGINT UNSIGNED"); + break; + case TIMESTAMP: + sb.append("TIMESTAMP"); + break; + + default: + throw new AssertionError(cast.getType()); + } + } + + @Override + public void visit(CnosDBBetweenOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(") BETWEEN ("); + visit(op.getLeft()); + sb.append(") AND ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(CnosDBInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN ("); + visit(op.getListElements()); + sb.append(")"); + } + + @Override + public void visit(CnosDBPostfixText op) { + visit(op.getExpr()); + sb.append(op.getText()); + } + + @Override + public void visit(CnosDBAggregate op) { + sb.append(op.getFunction()); + sb.append("("); + visit(op.getArgs()); + sb.append(")"); + } + + @Override + public void visit(CnosDBSimilarTo op) { + sb.append("("); + visit(op.getString()); + sb.append(" SIMILAR TO "); + visit(op.getSimilarTo()); + sb.append(")"); + } + + @Override + public void visit(CnosDBBinaryLogicalOperation op) { + super.visit((BinaryOperation) op); + } + + @Override + public void visit(CnosDBLikeOperation op) { + super.visit((BinaryOperation) op); + } + +} diff --git a/src/sqlancer/cnosdb/CnosDBVisitor.java b/src/sqlancer/cnosdb/CnosDBVisitor.java new file mode 100644 index 000000000..7c1af7224 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBVisitor.java @@ -0,0 +1,102 @@ +package sqlancer.cnosdb; + +import sqlancer.cnosdb.ast.CnosDBAggregate; +import sqlancer.cnosdb.ast.CnosDBBetweenOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; +import sqlancer.cnosdb.ast.CnosDBCastOperation; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBFunction; +import sqlancer.cnosdb.ast.CnosDBInOperation; +import sqlancer.cnosdb.ast.CnosDBLikeOperation; +import sqlancer.cnosdb.ast.CnosDBOrderByTerm; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation; +import sqlancer.cnosdb.ast.CnosDBPostfixText; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; +import sqlancer.cnosdb.ast.CnosDBSimilarTo; + +public interface CnosDBVisitor { + + static String asString(CnosDBExpression expr) { + CnosDBToStringVisitor visitor = new CnosDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + void visit(CnosDBConstant constant); + + void visit(CnosDBPostfixOperation op); + + void visit(CnosDBColumnValue c); + + void visit(CnosDBPrefixOperation op); + + void visit(CnosDBSelect op); + + void visit(CnosDBOrderByTerm op); + + void visit(CnosDBFunction f); + + void visit(CnosDBCastOperation cast); + + void visit(CnosDBBetweenOperation op); + + void visit(CnosDBInOperation op); + + void visit(CnosDBPostfixText op); + + void visit(CnosDBAggregate op); + + void visit(CnosDBFromTable from); + + void visit(CnosDBSubquery subquery); + + void visit(CnosDBBinaryLogicalOperation op); + + void visit(CnosDBLikeOperation op); + + void visit(CnosDBSimilarTo op); + + default void visit(CnosDBExpression expression) { + if (expression instanceof CnosDBConstant) { + visit((CnosDBConstant) expression); + } else if (expression instanceof CnosDBPostfixOperation) { + visit((CnosDBPostfixOperation) expression); + } else if (expression instanceof CnosDBColumnValue) { + visit((CnosDBColumnValue) expression); + } else if (expression instanceof CnosDBPrefixOperation) { + visit((CnosDBPrefixOperation) expression); + } else if (expression instanceof CnosDBSelect) { + visit((CnosDBSelect) expression); + } else if (expression instanceof CnosDBOrderByTerm) { + visit((CnosDBOrderByTerm) expression); + } else if (expression instanceof CnosDBFunction) { + visit((CnosDBFunction) expression); + } else if (expression instanceof CnosDBCastOperation) { + visit((CnosDBCastOperation) expression); + } else if (expression instanceof CnosDBBetweenOperation) { + visit((CnosDBBetweenOperation) expression); + } else if (expression instanceof CnosDBInOperation) { + visit((CnosDBInOperation) expression); + } else if (expression instanceof CnosDBAggregate) { + visit((CnosDBAggregate) expression); + } else if (expression instanceof CnosDBPostfixText) { + visit((CnosDBPostfixText) expression); + } else if (expression instanceof CnosDBSimilarTo) { + visit((CnosDBSimilarTo) expression); + } else if (expression instanceof CnosDBFromTable) { + visit((CnosDBFromTable) expression); + } else if (expression instanceof CnosDBSubquery) { + visit((CnosDBSubquery) expression); + } else if (expression instanceof CnosDBLikeOperation) { + visit((CnosDBLikeOperation) expression); + } else { + throw new AssertionError(expression); + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAggregate.java b/src/sqlancer/cnosdb/ast/CnosDBAggregate.java new file mode 100644 index 000000000..df30717b4 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBAggregate.java @@ -0,0 +1,113 @@ +package sqlancer.cnosdb.ast; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBBugs; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; +import sqlancer.common.ast.FunctionNode; + +public class CnosDBAggregate extends FunctionNode + implements CnosDBExpression { + + public CnosDBAggregate(List args, CnosDBAggregateFunction func) { + super(func, args); + } + + public enum CnosDBAggregateFunction { + AVG(CnosDBDataType.DOUBLE), + MAX(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, + CnosDBDataType.UINT), + MIN(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, + CnosDBDataType.UINT), + COUNT(CnosDBDataType.INT) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.getRandomType() }; + } + }, + SUM(CnosDBDataType.INT, CnosDBDataType.DOUBLE, CnosDBDataType.UINT), APPROX_MEDIAN(CnosDBDataType.DOUBLE), + + VAR(CnosDBDataType.DOUBLE), VAR_SAMP(CnosDBDataType.DOUBLE), VAR_POP(CnosDBDataType.DOUBLE), + STDDEV(CnosDBDataType.DOUBLE), STDDEV_SAMP(CnosDBDataType.DOUBLE), STDDEV_POP(CnosDBDataType.DOUBLE), + COVAR(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + COVAR_SAMP(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.INT }; + } + }, + CORR(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + COVAR_POP(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + + APPROX_PERCENTILE_CONT(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + APPROX_PERCENTILE_CONT_WITH_WEIGHT(CnosDBDataType.DOUBLE) { + @Override + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; + } + }, + APPROX_DISTINCT(CnosDBDataType.UINT), GROUPING(CnosDBDataType.INT), ARRAY_AGG(CnosDBDataType.STRING); + + private final CnosDBDataType[] supportedReturnTypes; + + CnosDBAggregateFunction(CnosDBDataType... supportedReturnTypes) { + this.supportedReturnTypes = supportedReturnTypes.clone(); + } + + public static List getAggregates(CnosDBDataType type) { + List res = Stream.of(values()).filter(p -> p.supportsReturnType(type)) + .collect(Collectors.toList()); + if (CnosDBBugs.BUG786) { + res.removeAll(List.of(VAR, VAR_POP, VAR_SAMP, STDDEV, STDDEV_POP, STDDEV_SAMP, CORR, COVAR, COVAR_POP, + COVAR_SAMP, APPROX_PERCENTILE_CONT_WITH_WEIGHT, APPROX_DISTINCT, APPROX_PERCENTILE_CONT, + APPROX_PERCENTILE_CONT_WITH_WEIGHT, GROUPING, ARRAY_AGG)); + } + + return res; + } + + public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { + return new CnosDBDataType[] { returnType }; + } + + public boolean supportsReturnType(CnosDBDataType returnType) { + return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType) + || supportedReturnTypes.length == 0; + } + + public CnosDBDataType getRandomReturnType() { + if (supportedReturnTypes.length == 0) { + return Randomly.fromOptions(CnosDBDataType.getRandomType()); + } else { + return Randomly.fromOptions(supportedReturnTypes); + } + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAlias.java b/src/sqlancer/cnosdb/ast/CnosDBAlias.java new file mode 100644 index 000000000..86bba199f --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBAlias.java @@ -0,0 +1,35 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.common.visitor.UnaryOperation; + +public class CnosDBAlias implements UnaryOperation, CnosDBExpression { + + private final CnosDBExpression expr; + private final String alias; + + public CnosDBAlias(CnosDBExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + @Override + public CnosDBExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return " as " + alias; + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java new file mode 100644 index 000000000..d0addced1 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java @@ -0,0 +1,34 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public final class CnosDBBetweenOperation implements CnosDBExpression { + + private final CnosDBExpression expr; + private final CnosDBExpression left; + private final CnosDBExpression right; + + public CnosDBBetweenOperation(CnosDBExpression expr, CnosDBExpression left, CnosDBExpression right) { + this.expr = expr; + this.left = left; + this.right = right; + } + + public CnosDBExpression getExpr() { + return expr; + } + + public CnosDBExpression getLeft() { + return left; + } + + public CnosDBExpression getRight() { + return right; + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java new file mode 100644 index 000000000..acf3e93d5 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java @@ -0,0 +1,69 @@ +package sqlancer.cnosdb.ast; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; +import sqlancer.common.ast.BinaryOperatorNode; + +public class CnosDBBinaryArithmeticOperation extends BinaryOperatorNode + implements CnosDBExpression { + + public CnosDBBinaryArithmeticOperation(CnosDBExpression left, CnosDBExpression right, CnosDBBinaryOperator op) { + super(left, right, op); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.INT; + } + + public enum CnosDBBinaryOperator implements BinaryOperatorNode.Operator { + + ADDITION("+") { + }, + SUBTRACTION("-") { + }, + MULTIPLICATION("*") { + }, + DIVISION("/") { + + }, + MODULO("%") { + }, + EXPONENTIATION("^") { + }; + + private final String textRepresentation; + + CnosDBBinaryOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static CnosDBBinaryOperator getRandom(CnosDBDataType dataType) { + List ops = new ArrayList<>(Arrays.asList(values())); + switch (dataType) { + case DOUBLE: + case UINT: + case STRING: + ops.remove(EXPONENTIATION); + ops.remove(MODULO); + break; + default: + break; + } + + return Randomly.fromList(ops); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java new file mode 100644 index 000000000..af38849c9 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java @@ -0,0 +1,57 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator; +import sqlancer.common.ast.BinaryOperatorNode; + +public class CnosDBBinaryComparisonOperation + extends BinaryOperatorNode implements CnosDBExpression { + + public CnosDBBinaryComparisonOperation(CnosDBExpression left, CnosDBExpression right, + CnosDBBinaryComparisonOperator op) { + super(left, right, op); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + public enum CnosDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("=") { + }, + IS_DISTINCT("IS DISTINCT FROM") { + }, + IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { + }, + NOT_EQUALS("!=") { + }, + LESS("<") { + }, + LESS_EQUALS("<=") { + }, + GREATER(">") { + }, + GREATER_EQUALS(">=") { + + }; + + private final String textRepresentation; + + CnosDBBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static CnosDBBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(CnosDBBinaryComparisonOperator.values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java new file mode 100644 index 000000000..bad8a3b75 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java @@ -0,0 +1,33 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; +import sqlancer.common.ast.BinaryOperatorNode; + +public class CnosDBBinaryLogicalOperation extends BinaryOperatorNode + implements CnosDBExpression { + + public CnosDBBinaryLogicalOperation(CnosDBExpression left, CnosDBExpression right, BinaryLogicalOperator op) { + super(left, right, op); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + public enum BinaryLogicalOperator implements BinaryOperatorNode.Operator { + AND, OR; + + public static BinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java b/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java new file mode 100644 index 000000000..41db62d81 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java @@ -0,0 +1,60 @@ +package sqlancer.cnosdb.ast; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.cnosdb.CnosDBCompoundDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBCastOperation implements CnosDBExpression { + + private final CnosDBExpression expression; + private final CnosDBCompoundDataType type; + + public CnosDBCastOperation(CnosDBExpression expression, CnosDBCompoundDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + public static List canCastTo(CnosDBDataType dataType) { + List options = new ArrayList<>(Arrays.asList(CnosDBDataType.values())); + + switch (dataType) { + case UINT: + case BOOLEAN: + case DOUBLE: + options.remove(CnosDBDataType.TIMESTAMP); + break; + case TIMESTAMP: + options.remove(CnosDBDataType.BOOLEAN); + options.remove(CnosDBDataType.UINT); + options.remove(CnosDBDataType.DOUBLE); + break; + default: + break; + } + return options; + } + + @Override + public CnosDBDataType getExpressionType() { + return type.getDataType(); + } + + public CnosDBExpression getExpression() { + return expression; + } + + public CnosDBDataType getType() { + return type.getDataType(); + } + + public CnosDBCompoundDataType getCompoundType() { + return type; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java b/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java new file mode 100644 index 000000000..f90b6120f --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java @@ -0,0 +1,27 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBColumnValue implements CnosDBExpression { + + private final CnosDBColumn c; + + public CnosDBColumnValue(CnosDBColumn c) { + this.c = c; + } + + public static CnosDBColumnValue create(CnosDBColumn c) { + return new CnosDBColumnValue(c); + } + + @Override + public CnosDBDataType getExpressionType() { + return c.getType(); + } + + public CnosDBColumn getColumn() { + return c; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java b/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java new file mode 100644 index 000000000..6821f83b8 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java @@ -0,0 +1,22 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.common.ast.BinaryNode; + +public class CnosDBConcatOperation extends BinaryNode implements CnosDBExpression { + + public CnosDBConcatOperation(CnosDBExpression left, CnosDBExpression right) { + super(left, right); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.STRING; + } + + @Override + public String getOperatorRepresentation() { + return "||"; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConstant.java b/src/sqlancer/cnosdb/ast/CnosDBConstant.java new file mode 100644 index 000000000..42ecd3908 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBConstant.java @@ -0,0 +1,520 @@ +package sqlancer.cnosdb.ast; + +import java.math.BigDecimal; +import java.text.SimpleDateFormat; +import java.util.Date; + +import sqlancer.IgnoreMeException; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public abstract class CnosDBConstant implements CnosDBExpression { + + public static CnosDBConstant createNullConstant() { + return new CnosDBNullConstant(); + } + + public static CnosDBConstant createIntConstant(long val) { + return new IntConstant(val, false); + } + + public static CnosDBConstant createBooleanConstant(boolean val) { + return new BooleanConstant(val); + } + + public static CnosDBConstant createFalse() { + return createBooleanConstant(false); + } + + public static CnosDBConstant createTrue() { + return createBooleanConstant(true); + } + + public static CnosDBConstant createStringConstant(String string) { + return new StringConstant(string); + } + + public static CnosDBConstant createDoubleConstant(double val) { + return new DoubleConstant(val); + } + + public static CnosDBConstant createUintConstant(long val) { + return new IntConstant(val, true); + } + + public static CnosDBConstant createTimeStampConstant(long val) { + return new TimeStampConstant(val); + } + + public abstract String getTextRepresentation(); + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isString() { + return false; + } + + public boolean isNull() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asDouble() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isBoolean() { + return false; + } + + public abstract CnosDBConstant isEquals(CnosDBConstant rightVal); + + public boolean isInt() { + return false; + } + + protected abstract CnosDBConstant isLessThan(CnosDBConstant rightVal); + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract CnosDBConstant cast(CnosDBDataType type); + + public static class BooleanConstant extends CnosDBConstant { + + private final boolean value; + + public BooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return value ? "TRUE" : "FALSE"; + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return CnosDBConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(value == rightVal.cast(CnosDBDataType.BOOLEAN).asBoolean()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isString()) { + return isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); + } else { + assert rightVal.isBoolean(); + return CnosDBConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + switch (type) { + case BOOLEAN: + return this; + case INT: + return CnosDBConstant.createIntConstant(value ? 1 : 0); + case UINT: + return CnosDBConstant.createUintConstant(value ? 1 : 0); + case STRING: + return CnosDBConstant.createStringConstant(value ? "true" : "false"); + default: + return null; + } + } + + } + + public static class CnosDBNullConstant extends CnosDBConstant { + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + return CnosDBConstant.createNullConstant(); + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + return CnosDBConstant.createNullConstant(); + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + return CnosDBConstant.createNullConstant(); + } + } + + public static class StringConstant extends CnosDBConstant { + + private final String value; + + public StringConstant(String value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'", value.replace("'", "''")); + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(CnosDBDataType.INT).isEquals(rightVal.cast(CnosDBDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal.cast(CnosDBDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(CnosDBDataType.INT).isLessThan(rightVal.cast(CnosDBDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + if (type == CnosDBDataType.STRING) { + return this; + } + String s = value.trim(); + switch (type) { + case BOOLEAN: + try { + return CnosDBConstant.createBooleanConstant(Long.parseLong(s) != 0); + } catch (NumberFormatException ignored) { + } + switch (s.toUpperCase()) { + case "T": + case "TR": + case "TRU": + case "TRUE": + case "1": + case "YES": + case "YE": + case "Y": + case "ON": + return CnosDBConstant.createTrue(); + case "F": + case "FA": + case "FAL": + case "FALS": + case "FALSE": + case "N": + case "NO": + case "OF": + case "OFF": + default: + return CnosDBConstant.createFalse(); + } + case INT: + try { + return CnosDBConstant.createIntConstant(Long.parseLong(s)); + } catch (NumberFormatException e) { + return CnosDBConstant.createIntConstant(-1); + } + case UINT: + try { + return CnosDBConstant.createUintConstant(Long.parseUnsignedLong(s)); + } catch (NumberFormatException e) { + return CnosDBConstant.createUintConstant(0); + } + case DOUBLE: + try { + return CnosDBConstant.createDoubleConstant(Double.parseDouble(s)); + } catch (NumberFormatException e) { + return CnosDBConstant.createDoubleConstant(0.0); + } + + default: + return null; + } + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.STRING; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public String asString() { + return value; + } + + } + + public static class IntConstant extends CnosDBConstant { + + private final long val; + private final boolean unsigned; + + public IntConstant(long val, boolean unsigned) { + this.val = val; + this.unsigned = unsigned; + } + + @Override + public String getTextRepresentation() { + if (unsigned) { + return Long.toUnsignedString(val); + } else { + return String.valueOf(val); + } + } + + @Override + public CnosDBDataType getExpressionType() { + if (unsigned) { + return CnosDBDataType.UINT; + } + return CnosDBDataType.INT; + } + + @Override + public long asInt() { + return val; + } + + @Override + public double asDouble() { + return val; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); + } else if (rightVal.isInt()) { + return CnosDBConstant.createBooleanConstant(val == rightVal.asInt()); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.INT).asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); + } else if (rightVal.isBoolean()) { + throw new AssertionError(rightVal); + } else if (rightVal.getExpressionType() == CnosDBDataType.UINT) { + return CnosDBConstant.createBooleanConstant(Long.compareUnsigned(val, rightVal.asInt()) < 0); + } else if (rightVal.isString()) { + return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.INT).asInt()); + } else { + throw new IgnoreMeException(); + } + + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + switch (type) { + case BOOLEAN: + return CnosDBConstant.createBooleanConstant(val != 0); + case INT: + return CnosDBConstant.createIntConstant(val); + case STRING: + return CnosDBConstant.createStringConstant(String.valueOf(val)); + case UINT: + return CnosDBConstant.createUintConstant(val); + case DOUBLE: + return CnosDBConstant.createDoubleConstant(val); + default: + return null; + } + } + } + + public static class TimeStampConstant extends CnosDBConstant { + final long val; + + TimeStampConstant(long time) { + val = time; + } + + @Override + public String getTextRepresentation() { + return "CAST (" + val + " AS TIMESTAMP)"; + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return createNullConstant(); + } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { + return createBooleanConstant(val == rightVal.asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { + return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + switch (type) { + case INT: + return createIntConstant(val); + case STRING: + final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); + return CnosDBConstant.createStringConstant(dateFormat.format(new Date(val))); + default: + return null; + } + } + + @Override + public long asInt() { + return val; + } + + } + + public static class DoubleConstant extends CnosDBConstant { + + private final double val; + + public DoubleConstant(double val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + BigDecimal bigDecimal = new BigDecimal(val); + return bigDecimal.toPlainString(); + } else { + return String.valueOf(0.0); + } + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.DOUBLE; + } + + @Override + public boolean isNull() { + return false; + } + + @Override + protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal); + } else { + return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); + } + } + + @Override + public CnosDBConstant isEquals(CnosDBConstant rightVal) { + if (rightVal.isNull()) { + return CnosDBConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); + } else { + return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); + } + } + + @Override + public CnosDBConstant cast(CnosDBDataType type) { + return null; + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBExpression.java b/src/sqlancer/cnosdb/ast/CnosDBExpression.java new file mode 100644 index 000000000..63997a0f5 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBExpression.java @@ -0,0 +1,14 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public interface CnosDBExpression { + + default CnosDBDataType getExpressionType() { + return null; + } + + default CnosDBConstant getExpectedValue() { + throw new AssertionError("Not impl"); + } +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunction.java b/src/sqlancer/cnosdb/ast/CnosDBFunction.java new file mode 100644 index 000000000..7a35d703e --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBFunction.java @@ -0,0 +1,30 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBFunction implements CnosDBExpression { + + private final String func; + private final CnosDBExpression[] args; + private final CnosDBDataType returnType; + + public CnosDBFunction(CnosDBFunctionWithUnknownResult f, CnosDBDataType returnType, CnosDBExpression... args) { + this.func = f.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public String getFunctionName() { + return func; + } + + public CnosDBExpression[] getArguments() { + return args.clone(); + } + + @Override + public CnosDBDataType getExpressionType() { + return returnType; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java b/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java new file mode 100644 index 000000000..485f2309d --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java @@ -0,0 +1,104 @@ +package sqlancer.cnosdb.ast; + +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.cnosdb.CnosDBBugs; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; + +public enum CnosDBFunctionWithUnknownResult { + + // String functions + ASCII("ascii", CnosDBDataType.INT, CnosDBDataType.STRING), + BTRIM("btrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + CHAR_LENGTH("char_length", CnosDBDataType.INT, CnosDBDataType.STRING), + CHARACTER_LENGTH("character_length", CnosDBDataType.INT, CnosDBDataType.STRING), + CONCAT("concat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + CONCAT_WS("concat_ws", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + CHR("chr", CnosDBDataType.STRING, CnosDBDataType.INT), + BIT_LENGTH("bit_length", CnosDBDataType.INT, CnosDBDataType.STRING), + INITCAP("initcap", CnosDBDataType.STRING, CnosDBDataType.STRING), + + LEFT("left", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + LENGTH("length", CnosDBDataType.UINT, CnosDBDataType.STRING), + LOWER("lower", CnosDBDataType.STRING, CnosDBDataType.STRING), + UPPER("upper", CnosDBDataType.STRING, CnosDBDataType.STRING), + LPAD3("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), + LPAD2("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + RPAD3("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), + RPAD2("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + LTRIM("ltrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + OCTET_LENGTH("octet_length", CnosDBDataType.INT, CnosDBDataType.STRING), + // REPEAT("repeat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + REPLACE("replace", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + REVERSE("reverse", CnosDBDataType.STRING, CnosDBDataType.STRING), + RIGHT("right", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + RTRIM("rtrim", CnosDBDataType.STRING, CnosDBDataType.STRING), + SPLIT_PART("split_part", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), + STARTS_WITH("starts_with", CnosDBDataType.BOOLEAN, CnosDBDataType.STRING, CnosDBDataType.STRING), + STRPOS("strpos", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.STRING), + SUBSTR("substr", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.INT), + TRANSLATE("translate", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), + MD5("md5", CnosDBDataType.STRING, CnosDBDataType.STRING), + // mathematical functions + ABS("abs", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + CEIL("ceil", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + EXP("exp", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), LN("ln", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + LOG2("log2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + LOG10("log10", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + POWER("power", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + ROUND("round", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + TRUNC("trunc", CnosDBDataType.DOUBLE, CnosDBDataType.INT), + FLOOR("floor", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + SIGNUM("signum", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + ACOS("acos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + ASIN("asin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + ATAN2("atan2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + COS("cos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), SIN("sin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + SQRT("sqrt", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + TAN("tan", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), + DATE_PART("date_part", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP), + TO_TIMESTAMP("to_timestamp", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), + TO_TIMESTAMP_MILLIS("to_timestamp_millis", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), + TO_TIMESTAMP_MICROS("to_timestamp_micros", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), + TO_TIMESTAMP_SECONDS("to_timestamp_seconds", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT); + + private final String functionName; + private final CnosDBDataType returnType; + private final CnosDBDataType[] argTypes; + + CnosDBFunctionWithUnknownResult(String functionName, CnosDBDataType returnType, CnosDBDataType... indexType) { + this.functionName = functionName; + this.returnType = returnType; + this.argTypes = indexType.clone(); + + } + + public static List getSupportedFunctions(CnosDBDataType type) { + List res = Stream.of(values()) + .filter(function -> function.isCompatibleWithReturnType(type)).collect(Collectors.toList()); + if (CnosDBBugs.BUG3547) { + res.removeAll(List.of(TO_TIMESTAMP, TO_TIMESTAMP_MICROS, TO_TIMESTAMP_MILLIS, TO_TIMESTAMP_SECONDS)); + } + return res; + } + + public boolean isCompatibleWithReturnType(CnosDBDataType t) { + return t == returnType; + } + + public CnosDBExpression[] getArguments(CnosDBDataType ignore, CnosDBExpressionGenerator gen, int depth) { + CnosDBExpression[] args = new CnosDBExpression[argTypes.length]; + for (int i = 0; i < args.length; i++) { + args[i] = gen.generateExpression(depth, argTypes[i]); + } + return args; + } + + public String getName() { + return functionName; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBInOperation.java b/src/sqlancer/cnosdb/ast/CnosDBInOperation.java new file mode 100644 index 000000000..c0ffd34ed --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBInOperation.java @@ -0,0 +1,35 @@ +package sqlancer.cnosdb.ast; + +import java.util.List; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBInOperation implements CnosDBExpression { + + private final CnosDBExpression expr; + private final List listElements; + private final boolean isTrue; + + public CnosDBInOperation(CnosDBExpression expr, List listElements, boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public CnosDBExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + public boolean isTrue() { + return isTrue; + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBJoin.java b/src/sqlancer/cnosdb/ast/CnosDBJoin.java new file mode 100644 index 000000000..eea88466f --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBJoin.java @@ -0,0 +1,46 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBJoin implements CnosDBExpression { + + private final CnosDBExpression tableReference; + private final CnosDBExpression onClause; + private final CnosDBJoinType type; + + public CnosDBJoin(CnosDBExpression tableReference, CnosDBExpression onClause, CnosDBJoinType type) { + this.tableReference = tableReference; + this.onClause = onClause; + this.type = type; + } + + public CnosDBExpression getTableReference() { + return tableReference; + } + + public CnosDBExpression getOnClause() { + return onClause; + } + + public CnosDBJoinType getType() { + return type; + } + + @Override + public CnosDBDataType getExpressionType() { + throw new AssertionError(); + } + + public enum CnosDBJoinType { + INNER, LEFT, RIGHT, FULL; + // now not support + // CROSS; + + public static CnosDBJoinType getRandom() { + return Randomly.fromOptions(values()); + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java b/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java new file mode 100644 index 000000000..616cd39ee --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java @@ -0,0 +1,22 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.common.ast.BinaryNode; + +public class CnosDBLikeOperation extends BinaryNode implements CnosDBExpression { + + public CnosDBLikeOperation(CnosDBExpression left, CnosDBExpression right) { + super(left, right); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + @Override + public String getOperatorRepresentation() { + return "LIKE"; + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java b/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java new file mode 100644 index 000000000..de5812d76 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java @@ -0,0 +1,37 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBOrderByTerm implements CnosDBExpression { + + private final CnosDBOrder order; + private final CnosDBExpression expr; + + public CnosDBOrderByTerm(CnosDBExpression expr, CnosDBOrder order) { + this.expr = expr; + this.order = order; + } + + public CnosDBOrder getOrder() { + return order; + } + + public CnosDBExpression getExpr() { + return expr; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + + public enum CnosDBOrder { + ASC, DESC; + + public static CnosDBOrder getRandomOrder() { + return Randomly.fromOptions(CnosDBOrder.values()); + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java new file mode 100644 index 000000000..f37621f44 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java @@ -0,0 +1,97 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.common.ast.BinaryOperatorNode.Operator; + +public class CnosDBPostfixOperation implements CnosDBExpression { + + private final CnosDBExpression expr; + private final String operatorTextRepresentation; + + public CnosDBPostfixOperation(CnosDBExpression expr, PostfixOperator op) { + this.expr = expr; + this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); + } + + public static CnosDBExpression create(CnosDBExpression expr, PostfixOperator op) { + return new CnosDBPostfixOperation(expr, op); + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + public String getOperatorTextRepresentation() { + return operatorTextRepresentation; + } + + public CnosDBExpression getExpression() { + return expr; + } + + public enum PostfixOperator implements Operator { + IS_NULL("IS NULL"/* , "ISNULL" */) { + @Override + public CnosDBDataType[] getInputDataTypes() { + return CnosDBDataType.values(); + } + + }, + IS_UNKNOWN("IS UNKNOWN") { + @Override + public CnosDBDataType[] getInputDataTypes() { + return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; + } + }, + + IS_NOT_NULL("IS NOT NULL"/* "NOTNULL" */) { + + @Override + public CnosDBDataType[] getInputDataTypes() { + return CnosDBDataType.values(); + } + + }, + IS_NOT_UNKNOWN("IS NOT UNKNOWN") { + + @Override + public CnosDBDataType[] getInputDataTypes() { + return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; + } + }, + IS_TRUE("IS TRUE") { + @Override + public CnosDBDataType[] getInputDataTypes() { + return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; + } + + }, + IS_FALSE("IS FALSE") { + @Override + public CnosDBDataType[] getInputDataTypes() { + return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; + } + + }; + + private final String[] textRepresentations; + + PostfixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public static PostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract CnosDBDataType[] getInputDataTypes(); + + @Override + public String getTextRepresentation() { + return toString(); + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java new file mode 100644 index 000000000..241fab89a --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java @@ -0,0 +1,29 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBPostfixText implements CnosDBExpression { + + private final CnosDBExpression expr; + private final String text; + private final CnosDBDataType type; + + public CnosDBPostfixText(CnosDBExpression expr, String text, CnosDBDataType type) { + this.expr = expr; + this.text = text; + this.type = type; + } + + public CnosDBExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + @Override + public CnosDBDataType getExpressionType() { + return type; + } +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java new file mode 100644 index 000000000..db37f0089 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java @@ -0,0 +1,73 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.common.ast.BinaryOperatorNode.Operator; + +public class CnosDBPrefixOperation implements CnosDBExpression { + + private final CnosDBExpression expr; + private final PrefixOperator op; + + public CnosDBPrefixOperation(CnosDBExpression expr, PrefixOperator op) { + this.expr = expr; + this.op = op; + } + + @Override + public CnosDBDataType getExpressionType() { + return op.getExpressionType(); + } + + public CnosDBDataType[] getInputDataTypes() { + return op.dataTypes; + } + + public String getTextRepresentation() { + return op.textRepresentation; + } + + public CnosDBExpression getExpression() { + return expr; + } + + public enum PrefixOperator implements Operator { + NOT("NOT", CnosDBDataType.BOOLEAN) { + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + + }, + UNARY_PLUS("+", CnosDBDataType.INT) { + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.INT; + } + + }, + UNARY_MINUS("-", CnosDBDataType.INT) { + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.INT; + } + + }; + + private final String textRepresentation; + private final CnosDBDataType[] dataTypes; + + PrefixOperator(String textRepresentation, CnosDBDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract CnosDBDataType getExpressionType(); + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSelect.java b/src/sqlancer/cnosdb/ast/CnosDBSelect.java new file mode 100644 index 000000000..0db657f19 --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBSelect.java @@ -0,0 +1,102 @@ +package sqlancer.cnosdb.ast; + +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.common.ast.SelectBase; + +public class CnosDBSelect extends SelectBase implements CnosDBExpression { + + private SelectType selectOption = SelectType.ALL; + private List joinClauses = Collections.emptyList(); + private CnosDBExpression distinctOnClause; + + public void setSelectType(SelectType fromOptions) { + this.setSelectOption(fromOptions); + } + + public SelectType getSelectOption() { + return selectOption; + } + + public void setSelectOption(SelectType fromOptions) { + this.selectOption = fromOptions; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + + public List getJoinClauses() { + return joinClauses; + } + + public void setJoinClauses(List joinStatements) { + this.joinClauses = joinStatements; + + } + + public CnosDBExpression getDistinctOnClause() { + return distinctOnClause; + } + + public void setDistinctOnClause(CnosDBExpression distinctOnClause) { + if (selectOption != SelectType.DISTINCT) { + throw new IllegalArgumentException(); + } + this.distinctOnClause = distinctOnClause; + } + + public enum SelectType { + DISTINCT, ALL; + + public static SelectType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public static class CnosDBFromTable implements CnosDBExpression { + private final CnosDBTable t; + + public CnosDBFromTable(CnosDBTable t) { + this.t = t; + } + + public CnosDBTable getTable() { + return t; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + } + + public static class CnosDBSubquery implements CnosDBExpression { + private final CnosDBSelect s; + private final String name; + + public CnosDBSubquery(CnosDBSelect s, String name) { + this.s = s; + this.name = name; + } + + public CnosDBSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public CnosDBDataType getExpressionType() { + return null; + } + } + +} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java b/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java new file mode 100644 index 000000000..9e3467ada --- /dev/null +++ b/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java @@ -0,0 +1,28 @@ +package sqlancer.cnosdb.ast; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public class CnosDBSimilarTo implements CnosDBExpression { + + private final CnosDBExpression string; + private final CnosDBExpression similarTo; + + public CnosDBSimilarTo(CnosDBExpression string, CnosDBExpression similarTo) { + this.string = string; + this.similarTo = similarTo; + } + + public CnosDBExpression getString() { + return string; + } + + public CnosDBExpression getSimilarTo() { + return similarTo; + } + + @Override + public CnosDBDataType getExpressionType() { + return CnosDBDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/cnosdb/client/CnosDBClient.java b/src/sqlancer/cnosdb/client/CnosDBClient.java new file mode 100644 index 000000000..ccc9dcc16 --- /dev/null +++ b/src/sqlancer/cnosdb/client/CnosDBClient.java @@ -0,0 +1,110 @@ +package sqlancer.cnosdb.client; + +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.io.UnsupportedEncodingException; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.codec.binary.Base64; +import org.apache.http.HttpHeaders; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpUriRequest; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; + +import com.arangodb.internal.util.IOUtils; + +public class CnosDBClient { + private final String userName; + private final String password; + private final String host; + private final int port; + + private final String database; + private final CloseableHttpClient client; + + public CnosDBClient(String host, int port, String userName, String password, String database) { + this.host = host; + this.port = port; + this.userName = userName; + this.password = password; + this.database = database; + this.client = HttpClientBuilder.create().build(); + } + + private String url() { + return "http://" + host + ":" + port + "/api/v1/"; + } + + public String ping() throws Exception { + HttpGet httpGet = new HttpGet(this.url() + "ping"); + httpGet.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); + CloseableHttpResponse resp = client.execute(httpGet); + + String content = IOUtils.toString(resp.getEntity().getContent()); + resp.close(); + return content; + } + + public CnosDBResultSet executeQuery(String query) throws Exception { + HttpUriRequest request = createRequest(query); + CloseableHttpResponse resp = client.execute(request); + String text = IOUtils.toString(resp.getEntity().getContent()); + if (resp.getStatusLine().getStatusCode() != 200) { + resp.close(); + throw new CnosDBException(database + ":" + query + ";\n" + text); + } + resp.close(); + InputStream stream = new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); + + return new CnosDBResultSet(new InputStreamReader(stream)); + } + + public boolean execute(String query) throws Exception { + HttpUriRequest request = createRequest(query); + CloseableHttpResponse resp = client.execute(request); + if (resp.getStatusLine().getStatusCode() != 200) { + String res = IOUtils.toString(resp.getEntity().getContent()); + resp.close(); + throw new CnosDBException(query + res); + } + resp.close(); + return true; + } + + public void close() throws IOException { + client.close(); + } + + public String getDatabase() { + return this.database; + } + + private String getAuth() { + String auth = userName + ":" + password; + byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.ISO_8859_1)); + return "Basic " + new String(encodedAuth); + + } + + private HttpUriRequest createRequest(String query) throws URISyntaxException, UnsupportedEncodingException { + + URIBuilder builder = new URIBuilder(this.url() + "sql"); + builder.setParameter("db", database); + builder.setParameter("pretty", "true"); + HttpPost httpPost = new HttpPost(builder.build()); + + httpPost.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); + StringEntity stringEntity = new StringEntity(query); + httpPost.setEntity(stringEntity); + return httpPost; + } + +} diff --git a/src/sqlancer/cnosdb/client/CnosDBConnection.java b/src/sqlancer/cnosdb/client/CnosDBConnection.java new file mode 100644 index 000000000..9277f203b --- /dev/null +++ b/src/sqlancer/cnosdb/client/CnosDBConnection.java @@ -0,0 +1,27 @@ +package sqlancer.cnosdb.client; + +import java.io.IOException; + +import sqlancer.SQLancerDBConnection; + +public class CnosDBConnection implements SQLancerDBConnection { + private final CnosDBClient client; + + public CnosDBConnection(CnosDBClient client) { + this.client = client; + } + + @Override + public String getDatabaseVersion() throws Exception { + return client.ping(); + } + + public CnosDBClient getClient() { + return client; + } + + @Override + public void close() throws IOException { + client.close(); + } +} diff --git a/src/sqlancer/cnosdb/client/CnosDBException.java b/src/sqlancer/cnosdb/client/CnosDBException.java new file mode 100644 index 000000000..a1055e90b --- /dev/null +++ b/src/sqlancer/cnosdb/client/CnosDBException.java @@ -0,0 +1,9 @@ +package sqlancer.cnosdb.client; + +public class CnosDBException extends RuntimeException { + private static final long serialVersionUID = 1L; + + CnosDBException(String message) { + super(message); + } +} diff --git a/src/sqlancer/cnosdb/client/CnosDBResultSet.java b/src/sqlancer/cnosdb/client/CnosDBResultSet.java new file mode 100644 index 000000000..877b6ba5d --- /dev/null +++ b/src/sqlancer/cnosdb/client/CnosDBResultSet.java @@ -0,0 +1,52 @@ +package sqlancer.cnosdb.client; + +import java.io.Reader; +import java.sql.SQLException; +import java.util.Iterator; + +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVRecord; + +import sqlancer.IgnoreMeException; + +public class CnosDBResultSet { + private final Iterator records; + private CSVRecord next; + + public CnosDBResultSet(Reader in) throws Exception { + Iterable records = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).build() + .parse(in); + this.records = records.iterator(); + } + + public void close() { + } + + public boolean next() throws SQLException { + if (records.hasNext()) { + next = records.next(); + return true; + } + return false; + } + + public int getInt(int i) throws SQLException { + return Integer.parseInt(next.get(i - 1)); + } + + public String getString(int i) throws SQLException { + return next.get(i - 1); + } + + public long getLong(int i) throws SQLException { + if (next.get(i - 1).isEmpty()) { + throw new IgnoreMeException(); + } + return Long.parseLong(next.get(i - 1)); + } + + // public boolean getBool(int i) throws Exception { + // return Boolean.parseBoolean(getString(i)); + // } + +} diff --git a/src/sqlancer/cnosdb/gen/CnosDBCommon.java b/src/sqlancer/cnosdb/gen/CnosDBCommon.java new file mode 100644 index 000000000..6c7b0bba7 --- /dev/null +++ b/src/sqlancer/cnosdb/gen/CnosDBCommon.java @@ -0,0 +1,31 @@ +package sqlancer.cnosdb.gen; + +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; + +public final class CnosDBCommon { + + private CnosDBCommon() { + } + + public static void appendDataType(CnosDBDataType type, StringBuilder sb) throws AssertionError { + switch (type) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: + sb.append("BIGINT"); + break; + case STRING: + sb.append("STRING"); + break; + case DOUBLE: + sb.append("DOUBLE"); + break; + case UINT: + sb.append("BIGINT UNSIGNED"); + break; + default: + throw new AssertionError(type); + } + } +} diff --git a/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java new file mode 100644 index 000000000..121f78254 --- /dev/null +++ b/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java @@ -0,0 +1,461 @@ +package sqlancer.cnosdb.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBCompoundDataType; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.ast.CnosDBAggregate; +import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; +import sqlancer.cnosdb.ast.CnosDBBetweenOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; +import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; +import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; +import sqlancer.cnosdb.ast.CnosDBCastOperation; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBConcatOperation; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBFunction; +import sqlancer.cnosdb.ast.CnosDBFunctionWithUnknownResult; +import sqlancer.cnosdb.ast.CnosDBInOperation; +import sqlancer.cnosdb.ast.CnosDBLikeOperation; +import sqlancer.cnosdb.ast.CnosDBOrderByTerm; +import sqlancer.cnosdb.ast.CnosDBOrderByTerm.CnosDBOrder; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; +import sqlancer.cnosdb.ast.CnosDBSimilarTo; +import sqlancer.common.gen.ExpressionGenerator; + +public class CnosDBExpressionGenerator implements ExpressionGenerator { + + private final int maxDepth; + + private final Randomly r; + + private List columns; + + private boolean allowAggregateFunctions; + + public CnosDBExpressionGenerator(CnosDBGlobalState globalState) { + this.r = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + } + + public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, CnosDBDataType type) { + return new CnosDBExpressionGenerator(globalState).generateExpression(0, type); + } + + private static CnosDBCompoundDataType getCompoundDataType(CnosDBDataType type) { + return CnosDBCompoundDataType.create(type); + } + + public static CnosDBExpression generateConstant(Randomly r, CnosDBDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return CnosDBConstant.createNullConstant(); + } + switch (type) { + case INT: + return CnosDBConstant.createIntConstant(r.getInteger()); + case UINT: + return CnosDBConstant.createUintConstant(r.getPositiveInteger()); + case TIMESTAMP: + return CnosDBConstant.createTimeStampConstant(r.getPositiveIntegerNotNull()); + case BOOLEAN: + return CnosDBConstant.createBooleanConstant(Randomly.getBoolean()); + case STRING: + return CnosDBConstant.createStringConstant(r.getString()); + case DOUBLE: + return CnosDBConstant.createDoubleConstant(r.getDouble()); + default: + throw new AssertionError(type); + } + } + + public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns, + CnosDBDataType type) { + return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); + } + + public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns) { + return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0); + } + + public CnosDBExpressionGenerator setColumns(List columns) { + this.columns = columns; + return this; + } + + public CnosDBExpression generateExpression(int depth) { + return generateExpression(depth, CnosDBDataType.getRandomType()); + } + + public List generateOrderBy() { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new CnosDBOrderByTerm(CnosDBColumnValue.create(Randomly.fromList(columns)), + CnosDBOrder.getRandomOrder())); + } + return orderBys; + } + + private CnosDBExpression generateFunctionWithUnknownResult(int depth, CnosDBDataType type) { + List supportedFunctions = CnosDBFunctionWithUnknownResult + .getSupportedFunctions(type); + if (supportedFunctions.isEmpty()) { + throw new IgnoreMeException(); + } + CnosDBFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); + return new CnosDBFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); + } + + private CnosDBExpression generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + PostfixOperator random = PostfixOperator.getRandom(); + return CnosDBPostfixOperation + .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); + case IN_OPERATION: + return inOperation(depth + 1); + case NOT: + return new CnosDBPrefixOperation(generateExpression(depth + 1, CnosDBDataType.BOOLEAN), PrefixOperator.NOT); + case BINARY_LOGICAL_OPERATOR: + CnosDBExpression first = generateExpression(depth + 1, CnosDBDataType.BOOLEAN); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + first = new CnosDBBinaryLogicalOperation(first, generateExpression(depth + 1, CnosDBDataType.BOOLEAN), + BinaryLogicalOperator.getRandom()); + } + return first; + case BINARY_COMPARISON: + CnosDBDataType dataType = getMeaningfulType(); + return generateComparison(depth, dataType); + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.BOOLEAN); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.BOOLEAN); + case LIKE: + return new CnosDBLikeOperation(generateExpression(depth + 1, CnosDBDataType.STRING), + generateExpression(depth + 1, CnosDBDataType.STRING)); + case BETWEEN: + CnosDBDataType type = getMeaningfulType(); + return new CnosDBBetweenOperation(generateExpression(depth + 1, type), generateExpression(depth + 1, type), + generateExpression(depth + 1, type)); + case SIMILAR_TO: + return new CnosDBSimilarTo(generateExpression(depth + 1, CnosDBDataType.STRING), + generateExpression(depth + 1, CnosDBDataType.STRING)); + default: + throw new AssertionError(); + } + } + + private CnosDBDataType getMeaningfulType() { + // make it more likely that the expression does not only consist of constant + // expressions + if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { + return CnosDBDataType.getRandomType(); + } else { + return Randomly.fromList(columns).getType(); + } + } + + private CnosDBExpression generateFunction(int depth, CnosDBDataType type) { + return generateFunctionWithUnknownResult(depth, type); + } + + private CnosDBExpression generateComparison(int depth, CnosDBDataType dataType) { + CnosDBExpression leftExpr = generateExpression(depth + 1, dataType); + CnosDBExpression rightExpr = generateExpression(depth + 1, dataType); + return getComparison(leftExpr, rightExpr); + } + + private CnosDBExpression getComparison(CnosDBExpression leftExpr, CnosDBExpression rightExpr) { + return new CnosDBBinaryComparisonOperation(leftExpr, rightExpr, + CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator.getRandom()); + } + + private CnosDBExpression inOperation(int depth) { + CnosDBDataType type = CnosDBDataType.getRandomType(); + CnosDBExpression leftExpr = generateExpression(depth + 1, type); + List rightExpr = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + rightExpr.add(generateConstant(new Randomly(), type)); + } + return new CnosDBInOperation(leftExpr, rightExpr, Randomly.getBoolean()); + } + + public CnosDBExpression generateExpression(int depth, CnosDBDataType originalType) { + return generateExpressionInternal(depth, originalType); + } + + private CnosDBExpression generateExpressionInternal(int depth, CnosDBDataType dataType) throws AssertionError { + if (allowAggregateFunctions && Randomly.getBoolean()) { + return getAggregate(dataType); + } + + if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { + // generic expression + if (Randomly.getBoolean() || depth > maxDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(r, dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(r, dataType); + } else { + return createColumnOfType(dataType); + } + } + } else { + if (Randomly.getBoolean()) { + return generateCastExpression(depth + 1, dataType); + } else { + return generateFunctionWithUnknownResult(depth, dataType); + } + } + } else { + switch (dataType) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case UINT: + return generateUIntExpression(depth); + case STRING: + return generateStringExpression(depth); + case DOUBLE: + return generateFloatExpression(depth); + case TIMESTAMP: + return generateTimeStampExpression(depth); + default: + throw new AssertionError(dataType); + } + } + } + + private CnosDBExpression generateStringExpression(int depth) { + StringExpression option; + List validOptions = new ArrayList<>(Arrays.asList(StringExpression.values())); + option = Randomly.fromList(validOptions); + + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.STRING); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.STRING); + case CONCAT: + return generateConcat(depth); + default: + throw new AssertionError(); + } + } + + private CnosDBExpression generateConcat(int depth) { + CnosDBExpression left = generateExpression(depth + 1, CnosDBDataType.STRING); + CnosDBExpression right = generateExpression(depth + 1); + return new CnosDBConcatOperation(left, right); + } + + private CnosDBExpression generateIntExpression(int depth) { + IntExpression option; + option = Randomly.fromOptions(IntExpression.values()); + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.INT); + case UNARY_OPERATION: + CnosDBExpression intExpression = generateExpression(depth + 1, CnosDBDataType.INT); + return new CnosDBPrefixOperation(intExpression, + Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.INT); + case BINARY_ARITHMETIC_EXPRESSION: + return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.INT), + generateExpression(depth + 1, CnosDBDataType.INT), + CnosDBBinaryOperator.getRandom(CnosDBDataType.INT)); + default: + throw new AssertionError(); + } + } + + private CnosDBExpression generateUIntExpression(int depth) { + UIntExpression option = Randomly.fromOptions(UIntExpression.values()); + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.UINT); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.UINT); + case BINARY_ARITHMETIC_EXPRESSION: + return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.UINT), + generateExpression(depth + 1, CnosDBDataType.UINT), + CnosDBBinaryOperator.getRandom(CnosDBDataType.UINT)); + default: + throw new AssertionError(); + } + + } + + private CnosDBExpression generateFloatExpression(int depth) { + FloatExpression option; + option = Randomly.fromOptions(FloatExpression.values()); + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.DOUBLE); + case UNARY_OPERATION: + CnosDBExpression floatExpression = generateExpression(depth + 1, CnosDBDataType.DOUBLE); + return new CnosDBPrefixOperation(floatExpression, + Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.DOUBLE); + case BINARY_ARITHMETIC_EXPRESSION: + return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.DOUBLE), + generateExpression(depth + 1, CnosDBDataType.DOUBLE), + CnosDBBinaryOperator.getRandom(CnosDBDataType.DOUBLE)); + case CONSTANT: + return generateConstant(r, CnosDBDataType.DOUBLE); + default: + throw new AssertionError(); + } + } + + private CnosDBExpression generateTimeStampExpression(int depth) { + if (Randomly.getBoolean()) { + return generateConstant(r, CnosDBDataType.TIMESTAMP); + } + TimestampExpression option; + option = Randomly.fromOptions(TimestampExpression.values()); + switch (option) { + case CAST: + return generateCastExpression(depth + 1, CnosDBDataType.TIMESTAMP); + case FUNCTION: + return generateFunction(depth + 1, CnosDBDataType.TIMESTAMP); + default: + throw new AssertionError(); + } + } + + private CnosDBExpression generateCastExpression(int depth, CnosDBDataType castToType) { + CnosDBDataType castFromType = Randomly.fromList(CnosDBCastOperation.canCastTo(castToType)); + return new CnosDBCastOperation(generateExpression(depth + 1, castFromType), getCompoundDataType(castToType)); + } + + private CnosDBExpression createColumnOfType(CnosDBDataType type) { + List columns = filterColumns(type); + if (columns.isEmpty()) { + throw new IgnoreMeException(); + } + CnosDBColumn fromList = Randomly.fromList(columns); + return CnosDBColumnValue.create(fromList); + } + + final List filterColumns(CnosDBDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + public List generateExpressions(int nr) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(0)); + } + return expressions; + } + + public CnosDBExpression generateExpression(CnosDBDataType dataType) { + return generateExpression(0, dataType); + } + + public CnosDBExpression generateHavingClause() { + this.allowAggregateFunctions = true; + CnosDBExpression expression = generateExpression(CnosDBDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + + public CnosDBExpression generateAggregate() { + return getAggregate(CnosDBDataType.getRandomType()); + } + + private CnosDBExpression getAggregate(CnosDBDataType dataType) { + if (dataType == CnosDBDataType.BOOLEAN) { + List aggregates = CnosDBAggregateFunction.getAggregates(CnosDBDataType.INT); + CnosDBAggregateFunction agg = Randomly.fromList(aggregates); + return new CnosDBCastOperation(generateArgsForAggregate(dataType, agg), + CnosDBCompoundDataType.create(CnosDBDataType.BOOLEAN)); + } else { + List aggregates = CnosDBAggregateFunction.getAggregates(dataType); + CnosDBAggregateFunction agg = Randomly.fromList(aggregates); + return generateArgsForAggregate(dataType, agg); + } + } + + public CnosDBAggregate generateArgsForAggregate(CnosDBDataType dataType, CnosDBAggregateFunction agg) { + CnosDBDataType[] types = agg.getArgsTypes(dataType); + List args = new ArrayList<>(); + for (CnosDBDataType argType : types) { + args.add(createColumnOfType(argType)); + // args.add(generateExpression(argType)); + } + return new CnosDBAggregate(args, agg); + } + + public CnosDBExpressionGenerator allowAggregates(boolean value) { + allowAggregateFunctions = value; + return this; + } + + @Override + public CnosDBExpression generatePredicate() { + return generateExpression(CnosDBDataType.BOOLEAN); + } + + @Override + public CnosDBExpression negatePredicate(CnosDBExpression predicate) { + return new CnosDBPrefixOperation(predicate, PrefixOperator.NOT); + } + + @Override + public CnosDBExpression isNull(CnosDBExpression expr) { + return new CnosDBPostfixOperation(expr, PostfixOperator.IS_NULL); + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, CAST, LIKE, BETWEEN, IN_OPERATION, + SIMILAR_TO, + } + + private enum StringExpression { + CAST, FUNCTION, CONCAT + } + + private enum IntExpression { + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION + } + + private enum UIntExpression { + FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION + } + + private enum FloatExpression { + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, CONSTANT + } + + private enum TimestampExpression { + FUNCTION, CAST + } + +} diff --git a/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java new file mode 100644 index 000000000..0d575d3c7 --- /dev/null +++ b/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java @@ -0,0 +1,59 @@ +package sqlancer.cnosdb.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.cnosdb.CnosDBVisitor; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.query.CnosDBOtherQuery; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; + +public final class CnosDBInsertGenerator { + + private CnosDBInsertGenerator() { + } + + public static CnosDBOtherQuery insert(CnosDBGlobalState globalState) { + CnosDBTable table = globalState.getSchema().getRandomTable(); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("Column time cannot be null."); + StringBuilder sb = new StringBuilder(); + sb.append("INSERT "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES"); + + int n = Randomly.smallNumber() + 1; + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + insertRow(globalState, sb, columns); + } + + // error + return new CnosDBOtherQuery(sb.toString(), errors); + } + + private static void insertRow(CnosDBGlobalState globalState, StringBuilder sb, List columns) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i > 0) { + sb.append(", "); + } + CnosDBExpression generateConstant = CnosDBExpressionGenerator.generateConstant(globalState.getRandomly(), + columns.get(i).getType()); + sb.append(CnosDBVisitor.asString(generateConstant)); + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java new file mode 100644 index 000000000..c046ad3e9 --- /dev/null +++ b/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java @@ -0,0 +1,77 @@ +package sqlancer.cnosdb.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBFieldColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTagColumn; +import sqlancer.cnosdb.query.CnosDBOtherQuery; +import sqlancer.common.query.ExpectedErrors; + +public class CnosDBTableGenerator { + + protected final ExpectedErrors errors = new ExpectedErrors(); + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + private final List columnsToBeAdd = new ArrayList<>(); + private CnosDBTable table; + + public CnosDBTableGenerator(String tableName) { + this.tableName = tableName; + } + + public static CnosDBOtherQuery generate(String tableName) { + return new CnosDBTableGenerator(tableName).generate(); + } + + protected CnosDBOtherQuery generate() { + table = new CnosDBTable(tableName, columnsToBeAdd); + + sb.append("CREATE TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + + sb.append("("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String name = String.format("f%d", i); + createField(name); + sb.append(", "); + } + + sb.append("TAGS("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + String name = String.format("t%d", i); + createTag(name); + } + sb.append("))"); + return new CnosDBOtherQuery(sb.toString(), new ExpectedErrors()); + } + + private void createField(String name) throws AssertionError { + sb.append(name); + sb.append(" "); + CnosDBDataType type = CnosDBDataType.getRandomTypeWithoutTimeStamp(); + CnosDBCommon.appendDataType(type, sb); + CnosDBFieldColumn c = new CnosDBFieldColumn(name, type); + c.setTable(table); + sb.append(" "); + columnsToBeAdd.add(c); + } + + private void createTag(String name) { + sb.append(name); + CnosDBColumn column = new CnosDBTagColumn(name); + column.setTable(table); + columnsToBeAdd.add(column); + } +} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java new file mode 100644 index 000000000..472aa8f66 --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java @@ -0,0 +1,23 @@ +package sqlancer.cnosdb.oracle; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.common.oracle.TestOracle; + +public abstract class CnosDBNoRECBase implements TestOracle { + protected final CnosDBGlobalState state; + protected final Main.StateLogger logger; + protected final MainOptions options; + protected final CnosDBConnection con; + protected String optimizedQueryString; + protected String unoptimizedQueryString; + + public CnosDBNoRECBase(CnosDBGlobalState state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } +} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java new file mode 100644 index 000000000..df91e9a58 --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java @@ -0,0 +1,171 @@ +package sqlancer.cnosdb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBCompoundDataType; +import sqlancer.cnosdb.CnosDBExpectedError; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; +import sqlancer.cnosdb.CnosDBVisitor; +import sqlancer.cnosdb.ast.CnosDBCastOperation; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBJoin; +import sqlancer.cnosdb.ast.CnosDBJoin.CnosDBJoinType; +import sqlancer.cnosdb.ast.CnosDBPostfixText; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; +import sqlancer.cnosdb.ast.CnosDBSelect.SelectType; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPBase; +import sqlancer.cnosdb.query.CnosDBSelectQuery; +import sqlancer.common.oracle.TestOracle; + +public class CnosDBNoRECOracle extends CnosDBNoRECBase implements TestOracle { + + private final CnosDBSchema s; + + public CnosDBNoRECOracle(CnosDBGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + } + + public static List getJoinStatements(CnosDBGlobalState globalState, List columns, + List tables) { + List joinStatements = new ArrayList<>(); + CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); + CnosDBTable table = Randomly.fromList(tables); + tables.remove(table); + CnosDBJoinType options = CnosDBJoinType.getRandom(); + CnosDBJoin j = new CnosDBJoin(new CnosDBFromTable(table), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + CnosDBTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + CnosDBSubquery subquery = CnosDBTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); + CnosDBJoinType options = CnosDBJoinType.getRandom(); + CnosDBJoin j = new CnosDBJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public void check() throws Exception { + CnosDBTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + CnosDBExpression randomWhereCondition = getRandomWhereCondition(columns); + List tables = randomTables.getTables(); + + List joinStatements = getJoinStatements(state, columns, tables); + List fromTables = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); + int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); + int firstCount = getOptimizedQueryCount(fromTables, List.of(CnosDBColumn.createDummy("f0")), + randomWhereCondition, joinStatements); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, + firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + private CnosDBExpression getRandomWhereCondition(List columns) { + return new CnosDBExpressionGenerator(state).setColumns(columns).generateExpression(CnosDBDataType.BOOLEAN); + } + + private int getUnoptimizedQueryCount(List fromTables, CnosDBExpression randomWhereCondition, + List joinStatements) throws Exception { + CnosDBSelect select = new CnosDBSelect(); + CnosDBCastOperation isTrue = new CnosDBCastOperation(randomWhereCondition, + CnosDBCompoundDataType.create(CnosDBDataType.INT)); + CnosDBPostfixText asText = new CnosDBPostfixText(isTrue, " as count", CnosDBDataType.INT); + select.setFetchColumns(List.of(asText)); + select.setFromList(fromTables); + select.setSelectType(SelectType.ALL); + select.setJoinClauses(joinStatements); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + CnosDBVisitor.asString(select) + ") as res"; + if (options.logEachSelect()) { + logger.writeCurrent(unoptimizedQueryString); + } + CnosDBSelectQuery q = new CnosDBSelectQuery(unoptimizedQueryString, CnosDBExpectedError.expectedErrors()); + CnosDBResultSet rs; + try { + q.executeAndGet(state); + rs = q.getResultSet(); + } catch (Exception e) { + if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getOptimizedQueryCount(List randomTables, List columns, + CnosDBExpression randomWhereCondition, List joinStatements) { + CnosDBSelect select = new CnosDBSelect(); + CnosDBColumnValue allColumns = new CnosDBColumnValue(Randomly.fromList(columns)); + select.setFetchColumns(List.of(allColumns)); + select.setFromList(randomTables); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new CnosDBExpressionGenerator(state).setColumns(columns).generateOrderBy()); + } + select.setSelectType(SelectType.ALL); + select.setJoinClauses(joinStatements); + int firstCount = 0; + optimizedQueryString = CnosDBVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + CnosDBSelectQuery query = new CnosDBSelectQuery(optimizedQueryString, CnosDBExpectedError.expectedErrors()); + CnosDBResultSet rs; + try { + query.executeAndGet(state); + rs = query.getResultSet(); + while (rs.next()) { + firstCount++; + } + } catch (Exception e) { + if (query.getExpectedErrors().errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java new file mode 100644 index 000000000..6ff72558e --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java @@ -0,0 +1,176 @@ +package sqlancer.cnosdb.oracle.tlp; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBExpectedError; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBVisitor; +import sqlancer.cnosdb.ast.CnosDBAggregate; +import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; +import sqlancer.cnosdb.ast.CnosDBAlias; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBJoin; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation; +import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation; +import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.cnosdb.query.CnosDBSelectQuery; +import sqlancer.common.oracle.TestOracle; + +public class CnosDBTLPAggregateOracle extends CnosDBTLPBase implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public CnosDBTLPAggregateOracle(CnosDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() { + CnosDBAggregateFunction aggregateFunction = Randomly.fromOptions(CnosDBAggregateFunction.MAX, + CnosDBAggregateFunction.MIN, CnosDBAggregateFunction.SUM); + + CnosDBAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), + aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(fetchColumns); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + originalQuery = CnosDBVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + String queryFormatString = "-- %s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + String assertionMessage = String.format("%s: the results mismatch!\n%s\n%s", this.s.getDatabaseName(), + firstQueryString, secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + private String createMetamorphicUnionQuery(CnosDBSelect select, CnosDBAggregate aggregate, + List from) { + String metamorphicQuery; + CnosDBExpression whereClause = gen.generateExpression(CnosDBDataType.BOOLEAN); + CnosDBExpression negatedClause = new CnosDBPrefixOperation(whereClause, PrefixOperator.NOT); + CnosDBExpression notNullClause = new CnosDBPostfixOperation(whereClause, PostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + CnosDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); + CnosDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); + CnosDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += CnosDBVisitor.asString(leftSelect) + " UNION ALL " + CnosDBVisitor.asString(middleSelect) + + " UNION ALL " + CnosDBVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String resultString = null; + + CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, CnosDBExpectedError.expectedErrors()); + try { + q.executeAndGet(state); + CnosDBResultSet result = q.getResultSet(); + + if (result == null || !result.next()) { + throw new IgnoreMeException(); + } + + resultString = result.getString(1); + + } catch (Exception e) { + if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + } + + return resultString; + } + + private List mapped(CnosDBAggregate aggregate) { + switch (aggregate.getFunction()) { + case SUM: + case MAX: + case MIN: + return aliasArgs(List.of(aggregate)); + // now not support + // case COUNT: + // case AVG: + default: + throw new AssertionError(aggregate.getFunction()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (CnosDBExpression expr : originalAggregateArgs) { + args.add(new CnosDBAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(CnosDBAggregate aggregate) { + if (Objects.requireNonNull(aggregate.getFunction()) == CnosDBAggregateFunction.COUNT) { + return CnosDBAggregateFunction.SUM + "(agg0)"; + } + return aggregate.getFunction() + "(agg0)"; + } + + private CnosDBSelect getSelect(List aggregates, List from, + CnosDBExpression whereClause, List joinList) { + CnosDBSelect leftSelect = new CnosDBSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinClauses(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java new file mode 100644 index 000000000..a4dfd3061 --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java @@ -0,0 +1,112 @@ +package sqlancer.cnosdb.oracle.tlp; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema; +import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; +import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; +import sqlancer.cnosdb.ast.CnosDBColumnValue; +import sqlancer.cnosdb.ast.CnosDBConstant; +import sqlancer.cnosdb.ast.CnosDBExpression; +import sqlancer.cnosdb.ast.CnosDBJoin; +import sqlancer.cnosdb.ast.CnosDBSelect; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; +import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; +import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; +import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; + +public class CnosDBTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + protected CnosDBSchema s; + protected CnosDBTables targetTables; + protected CnosDBExpressionGenerator gen; + protected CnosDBSelect select; + + public CnosDBTLPBase(CnosDBGlobalState state) { + super(state); + } + + public static CnosDBSubquery createSubquery(CnosDBGlobalState globalState, String name, CnosDBTables tables) { + List columns = new ArrayList<>(); + CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + CnosDBSelect select = new CnosDBSelect(); + select.setFromList(tables.getTables().stream().map(CnosDBFromTable::new).collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, CnosDBDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + return new CnosDBSubquery(select, name); + } + + @Override + public void check() throws Exception { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + List tables = targetTables.getTables(); + List joins = getJoinStatements(targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + protected List getJoinStatements(List columns, List tables) { + return CnosDBNoRECOracle.getJoinStatements(state, columns, tables); + } + + protected void generateSelectBase(List tables, List joins) { + List tableList = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); + gen = new CnosDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new CnosDBSelect(); + select.setFetchColumns(generateFetchColumns()); + select.setFromList(tableList); + select.setWhereClause(null); + select.setJoinClauses(joins); + } + + List generateFetchColumns() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return List.of(new CnosDBColumnValue(CnosDBColumn.createDummy("*"))); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = targetTables.getRandomColumnsWithOnlyOneField(); + + ArrayList columns = new ArrayList<>(); + targetColumns.forEach(column -> column.getTable().getColumns().stream() + .filter(field -> field instanceof CnosDBSchema.CnosDBFieldColumn).findFirst().ifPresent(columns::add)); + targetColumns.addAll(columns); + + targetColumns = targetColumns.stream().distinct().collect(Collectors.toList()); + + for (CnosDBColumn c : targetColumns) { + fetchColumns.add(new CnosDBColumnValue(c)); + } + return fetchColumns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java new file mode 100644 index 000000000..39b611a81 --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java @@ -0,0 +1,65 @@ +package sqlancer.cnosdb.oracle.tlp; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBComparatorHelper; +import sqlancer.cnosdb.CnosDBExpectedError; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; +import sqlancer.cnosdb.CnosDBVisitor; +import sqlancer.cnosdb.ast.CnosDBExpression; + +public class CnosDBTLPHavingOracle extends CnosDBTLPBase { + + public CnosDBTLPHavingOracle(CnosDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + havingCheck(); + } + + protected void havingCheck() throws Exception { + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(CnosDBDataType.BOOLEAN)); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = CnosDBVisitor.asString(select); + List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, + CnosDBExpectedError.expectedErrors(), state); + + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + select.setHavingClause(predicate); + String firstQueryString = CnosDBVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = CnosDBVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = CnosDBVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, CnosDBExpectedError.expectedErrors()); + CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + protected CnosDBExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List generateFetchColumns() { + List expressions = gen.allowAggregates(true).generateExpressions(Randomly.smallNumber() + 1); + gen.allowAggregates(false); + return expressions; + } + +} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java new file mode 100644 index 000000000..3b3b6e4dd --- /dev/null +++ b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java @@ -0,0 +1,46 @@ +package sqlancer.cnosdb.oracle.tlp; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.cnosdb.CnosDBComparatorHelper; +import sqlancer.cnosdb.CnosDBExpectedError; +import sqlancer.cnosdb.CnosDBGlobalState; +import sqlancer.cnosdb.CnosDBVisitor; + +public class CnosDBTLPWhereOracle extends CnosDBTLPBase { + + public CnosDBTLPWhereOracle(CnosDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + whereCheck(); + } + + protected void whereCheck() throws Exception { + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + String originalQueryString = CnosDBVisitor.asString(select); + List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, + CnosDBExpectedError.expectedErrors(), state); + + select.setOrderByExpressions(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = CnosDBVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = CnosDBVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = CnosDBVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, CnosDBExpectedError.expectedErrors()); + CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } +} diff --git a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java new file mode 100644 index 000000000..bd48c7b7e --- /dev/null +++ b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java @@ -0,0 +1,30 @@ +package sqlancer.cnosdb.query; + +import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.common.query.ExpectedErrors; + +public class CnosDBOtherQuery extends CnosDBQueryAdapter { + public CnosDBOtherQuery(String query, ExpectedErrors errors) { + super(query, errors); + } + + @Override + public boolean couldAffectSchema() { + return true; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + try { + globalState.getConnection().getClient().execute(query); + } catch (Exception e) { + if (this.errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + } + return true; + } +} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java new file mode 100644 index 000000000..2b2dda5e6 --- /dev/null +++ b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java @@ -0,0 +1,41 @@ +package sqlancer.cnosdb.query; + +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; + +public abstract class CnosDBQueryAdapter extends Query { + + String query; + ExpectedErrors errors; + + public CnosDBQueryAdapter(String query, ExpectedErrors errors) { + this.query = query; + this.errors = errors; + } + + @Override + public String getLogString() { + return query; + } + + @Override + public String getQueryString() { + return query; + } + + @Override + public String getUnterminatedQueryString() { + return null; + } + + @Override + public boolean couldAffectSchema() { + return false; + } + + @Override + public ExpectedErrors getExpectedErrors() { + return errors; + } +} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java b/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java new file mode 100644 index 000000000..dee38abf4 --- /dev/null +++ b/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java @@ -0,0 +1,6 @@ +package sqlancer.cnosdb.query; + +@FunctionalInterface +public interface CnosDBQueryProvider { + CnosDBOtherQuery getQuery(S globalState) throws Exception; +} diff --git a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java new file mode 100644 index 000000000..ca7f61c3a --- /dev/null +++ b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java @@ -0,0 +1,38 @@ +package sqlancer.cnosdb.query; + +import sqlancer.GlobalState; +import sqlancer.cnosdb.client.CnosDBConnection; +import sqlancer.cnosdb.client.CnosDBResultSet; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; + +public class CnosDBSelectQuery extends CnosDBQueryAdapter { + CnosDBResultSet resultSet; + + public CnosDBSelectQuery(String query, ExpectedErrors errors) { + super(query, errors); + } + + @Override + public boolean couldAffectSchema() { + return false; + } + + @Override + public > boolean execute(G globalState, String... fills) + throws Exception { + globalState.getConnection().getClient().execute(query); + return false; + } + + @Override + public > SQLancerResultSet executeAndGet(G globalState, + String... fills) throws Exception { + resultSet = globalState.getConnection().getClient().executeQuery(query); + return null; + } + + public CnosDBResultSet getResultSet() { + return resultSet; + } +} From f2d3d823296ea8ee0cacf905cb2b8d965ed15ab6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 22:28:58 +0800 Subject: [PATCH 307/856] Add a FAQ section Address a question that frequently comes up (e.g., see https://github.com/sqlancer/sqlancer/issues/704 or https://github.com/sqlancer/sqlancer/issues/684). Also mention the locale issue (see https://github.com/sqlancer/sqlancer/issues/533). --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 283efcd29..780b11025 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,13 @@ We would appreciate it if you mention SQLancer when you report bugs found by it. We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. SQLancer's official Twitter handle is [@sqlancer_dbms](https://twitter.com/sqlancer_dbms). +# FAQ + +## I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors? + +In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) directory to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). + +Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html). # Additional Documentation From 44ead10dcfde8111a6d4ea86653e4d9362d77daa Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 22:32:55 +0800 Subject: [PATCH 308/856] Fix minor doc issues --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 780b11025..1fc7cdf06 100644 --- a/README.md +++ b/README.md @@ -93,9 +93,9 @@ We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_inv ## I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors? -In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) directory to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). +In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) file to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). -Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html). +Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html)). # Additional Documentation From 378f82ad76a1121a57695e6f84de66e23cc89fc7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 22:37:30 +0800 Subject: [PATCH 309/856] No longer mention that we will soon release the reducer --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1fc7cdf06..96e1985c6 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ SQLancer stores logs in the `target/logs` subdirectory. By default, the option ` ## Reducing a Bug -After finding a bug, it is useful to produce a minimal test case before reporting the bug, to save the DBMS developers' time and effort. For many test cases, [C-Reduce](https://embed.cs.utah.edu/creduce/) does a great job. In addition, we have been working on a SQL-specific reducer, which we plan to release soon. +After finding a bug, it is useful to produce a minimal test case before reporting the bug, to save the DBMS developers' time and effort. For many test cases, [C-Reduce](https://embed.cs.utah.edu/creduce/) does a great job. ## Found Bugs From 403ab6f6d4ff9e926e2e675d0e0f726739f550e2 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 22:46:01 +0800 Subject: [PATCH 310/856] Describe that a "test" database might be required For example, see https://github.com/sqlancer/sqlancer/issues/677 for this question. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 96e1985c6..047a30012 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,10 @@ In many cases, SQLancer does not support the latest version of a DBMS. You can c Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html)). +## When starting SQLancer, I get an error such as "database 'test' does not exist". How can I run SQLancer without this error? + +For some DBMSs, SQLancer expects that a database "test" exists, which it then uses as an initial database to connect to. If you have not yet created such a database, you can use a command such as `CREATE DATABASE test` to create this database (e.g., see the [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-createdatabase.html)). + # Additional Documentation * [Contributing to SQLancer](CONTRIBUTING.md) From a4a21c00129a276b7978599d574bdda9f5bb02f9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 28 Jan 2023 23:05:27 +0800 Subject: [PATCH 311/856] Add additional suggestions for contributors --- CONTRIBUTING.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b28a1be29..08749f141 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,6 +14,13 @@ If you do not find an option to import Maven projects, you might need to install The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see `DuckDBNoRECOracle` or `DuckDBQueryPartitioningWhereTester` for TLP). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). +Please consider the following suggestions when creating a PR to contribute a new DBMS: +* Ensure that `mvn verify -DskipTests=true` does not result in style violations. +* Add a [CI test](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) to ensure that future changes to SQLancer are unlikely to break the newly-supported DBMS. It is reasonable to do this in a follow-up PR—please indicate whether you plan to do so in the PR description. +* Add the DBMS' name to the [check_names.py](https://github.com/sqlancer/sqlancer/blob/master/src/check_names.py) script, which ensures adherence to a common prefix in the Java classes. +* Add the DBMS' name to the [README.md](https://github.com/sqlancer/sqlancer/blob/master/README.md#supported-dbms) file. +* It would be easier to review multiple smaller PRs, than one PR that contains the complete implementation. Consider contributing parts of your implementation as you work on their implementation. + ### Typed vs. Untyped Expression Generation Each DBMS implementation provides an expression generator used, for example, to generate expressions used in `WHERE` clauses. We found that DBMS can be roughly classified into "permissive" ones, which apply implicit type conversions when needed and "strict" ones, which provide only few implicit conversions and output an error when the type is unexpected. For example, consider the following test case: From 5f5ffb5c2717caa2f374cccc11102b9e2da9aa8e Mon Sep 17 00:00:00 2001 From: ZuoTiJia <49109132+ZuoTiJia@users.noreply.github.com> Date: Sun, 29 Jan 2023 11:06:52 +0800 Subject: [PATCH 312/856] Add CnosDB Info (#705) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 047a30012..346209908 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | | Databend | Working | Typed | | | QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | +| CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | # Using SQLancer From 5ec6334ab0622c37a3c43cc28187eb545eda3fe7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 29 Jan 2023 13:36:37 +0800 Subject: [PATCH 313/856] Ignore a known Citus bug (#706) See https://github.com/citusdata/citus/issues/6298. Partly addresses https://github.com/sqlancer/sqlancer/issues/698. --- src/sqlancer/citus/CitusBugs.java | 3 +++ src/sqlancer/citus/gen/CitusInsertGenerator.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/sqlancer/citus/CitusBugs.java b/src/sqlancer/citus/CitusBugs.java index bc81b9db6..a6f4910e6 100644 --- a/src/sqlancer/citus/CitusBugs.java +++ b/src/sqlancer/citus/CitusBugs.java @@ -30,6 +30,9 @@ public final class CitusBugs { // https://github.com/citusdata/citus/issues/4079 public static boolean bug4079 = true; + // https://github.com/citusdata/citus/issues/6298 + public static boolean bug6298 = true; + private CitusBugs() { } diff --git a/src/sqlancer/citus/gen/CitusInsertGenerator.java b/src/sqlancer/citus/gen/CitusInsertGenerator.java index 269d94bbe..78b5794e7 100644 --- a/src/sqlancer/citus/gen/CitusInsertGenerator.java +++ b/src/sqlancer/citus/gen/CitusInsertGenerator.java @@ -1,5 +1,6 @@ package sqlancer.citus.gen; +import sqlancer.citus.CitusBugs; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -14,6 +15,9 @@ public static SQLQueryAdapter insert(PostgresGlobalState globalState) { SQLQueryAdapter insertQuery = PostgresInsertGenerator.insert(globalState); ExpectedErrors errors = insertQuery.getExpectedErrors(); CitusCommon.addCitusErrors(errors); + if (CitusBugs.bug6298) { + errors.add("columnar_tuple_insert_speculative not implemented"); + } return insertQuery; } From 080eb773924440a8c4ba72167d50d0678b87fd0c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 29 Jan 2023 13:43:05 +0800 Subject: [PATCH 314/856] Ignore invalid datetime casts in TiDB (#708) Partly addresses https://github.com/sqlancer/sqlancer/issues/669. --- src/sqlancer/tidb/TiDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 0022908a9..c2db11757 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -13,6 +13,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("BIGINT UNSIGNED value is out of range"); errors.add("Data truncation: Truncated incorrect time value"); errors.add("Data truncation: Incorrect time value"); + errors.add("Data truncation: Incorrect datetime value"); errors.add("overflows double"); errors.add("overflows bigint"); errors.add("strconv.ParseFloat: parsing"); From 6d90c985950f4afbf12728d1f2cf99c00dc2a86d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 29 Jan 2023 14:14:04 +0800 Subject: [PATCH 315/856] Check the naming conventions of all current database systems (#709) --- src/check_names.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/check_names.py b/src/check_names.py index 72191ac7d..31d6853a4 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -18,12 +18,21 @@ def verify_prefix(prefix, files): print('The class name of ' + f + ' does not start with ' + prefix) exit(-1) +verify_prefix('ArangoDB', get_java_files("sqlancer/arangodb/")) +verify_prefix('Citus', get_java_files("sqlancer/citus/")) verify_prefix('ClickHouse', get_java_files("sqlancer/clickhouse/")) +verify_prefix('CnosDB', get_java_files("sqlancer/cnosdb/")) verify_prefix('CockroachDB', get_java_files("sqlancer/cockroachdb/")) +verify_prefix('Cosmos', get_java_files("sqlancer/cosmos/")) +verify_prefix('Databend', get_java_files("sqlancer/databend/")) verify_prefix('DuckDB', get_java_files("sqlancer/duckdb")) +verify_prefix('H2', get_java_files("sqlancer/h2")) +verify_prefix('HSQLDB', get_java_files("sqlancer/hsqldb")) verify_prefix('MariaDB', get_java_files("sqlancer/mariadb/")) verify_prefix('MySQL', get_java_files("sqlancer/mysql/")) +verify_prefix('OceanBase', get_java_files("sqlancer/oceanbase/")) verify_prefix('Postgres', get_java_files("sqlancer/postgres/")) +verify_prefix('QuestDB', get_java_files("sqlancer/questdb/")) verify_prefix('SQLite3', get_java_files("sqlancer/sqlite3/")) verify_prefix('TiDB', get_java_files("sqlancer/tidb/")) -verify_prefix('CnosDB', get_java_files("sqlancer/cnosdb/")) +verify_prefix('Y', get_java_files("sqlancer/yugabyte/")) # has both YCQL and YSQL prefixes From 222ae47a0e897968b403de98a516d02ed5bafae8 Mon Sep 17 00:00:00 2001 From: everpcpc Date: Tue, 31 Jan 2023 12:19:46 +0800 Subject: [PATCH 316/856] fix: credential env for databend docker image (#713) --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1587aee78..329f108da 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -146,7 +146,7 @@ jobs: run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test - + databend: name: DBMS Tests (Databend) runs-on: ubuntu-latest @@ -154,8 +154,8 @@ jobs: databend: image: datafuselabs/databend env: - DATABEND_DEFAULT_USER: sqlancer - DATABEND_DEFAULT_PASSWORD: sqlancer + QUERY_DEFAULT_USER: sqlancer + QUERY_DEFAULT_PASSWORD: sqlancer ports: - 8000:8000 - 3307:3307 From e148652d1533b8f122c29c3d4bbb41c77b1747c9 Mon Sep 17 00:00:00 2001 From: Yisong Han Date: Wed, 1 Feb 2023 15:00:58 +0800 Subject: [PATCH 317/856] fix(databend):get data type error (#715) * add(databend): databend bugs Signed-off-by: Yisong Han * fix(databend):get data type error Signed-off-by: Yisong Han * fix: credential env for databend docker image (#713) * fix(databend):reduce unimportant error messages Signed-off-by: Yisong Han --------- Signed-off-by: Yisong Han Co-authored-by: everpcpc Co-authored-by: Manuel Rigger --- src/sqlancer/databend/DatabendBugs.java | 1 + src/sqlancer/databend/DatabendErrors.java | 9 ++++++++- src/sqlancer/databend/DatabendSchema.java | 3 +++ .../databend/ast/DatabendBinaryArithmeticOperation.java | 2 +- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index 336443511..eec2c39d0 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -14,6 +14,7 @@ public final class DatabendBugs { public static boolean bug9235 = true; // https://github.com/datafuselabs/databend/issues/9235 public static boolean bug9236 = true; // https://github.com/datafuselabs/databend/issues/9236 public static boolean bug9264 = true; // https://github.com/datafuselabs/databend/issues/9264 + public static boolean bug9806 = true; // https://github.com/datafuselabs/databend/issues/9806 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 8f9d2b942..5e813dcbd 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -9,9 +9,13 @@ private DatabendErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Division by zero"); + errors.add("divided by zero"); errors.add("/ by zero"); errors.add("ORDER BY position"); errors.add("GROUP BY position"); + errors.add("no overload satisfies `not(Float64 NULL)`"); // TODO databend不允许出现not(float)，而a/b为float + errors.add("no overload satisfies `not(Float64)`"); + errors.add("number overflowed while evaluating function"); // 表达式数值溢出 if (DatabendBugs.bug9162) { errors.add("downcast column error"); } @@ -30,7 +34,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (DatabendBugs.bug9264) { errors.add("assertion failed: offset + length <= self.length"); } - // errors.add("validity's length must be equal"); // bug + if (DatabendBugs.bug9806) { + errors.add("segment pruning failure"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE @@ -45,6 +51,7 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Division by zero"); errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); + errors.add("number overflowed while evaluating function `"); // 不能在int16类型column上插入int64的数据 } public static void addGroupByErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 24d47ec46..2e8e3c2bf 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -339,6 +339,9 @@ private static List getTableColumns(SQLConnection con, String ta while (rs.next()) { String columnName = rs.getString("column_name"); String dataType = rs.getString("data_type"); + if (dataType.contains("NULL")) { + dataType = dataType.substring(0, dataType.indexOf(' ')); + } boolean isNullable = rs.getBoolean("is_nullable"); // boolean isPrimaryKey = rs.getString("pk").contains("true"); boolean isPrimaryKey = false; // 没找到主键元数据 diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java index f052b3ccb..391597941 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -34,7 +34,7 @@ public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { return applyOperation(left, right, (l, r) -> l * r); } }, - DIVISION("/") { + DIVISION("/") { // TODO databend不允许出现not(float)，而a/b为float @Override public DatabendConstant apply(DatabendConstant left, DatabendConstant right) { return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); From 34424b79d532ddf972413f828cc30f6b23c1803f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Feb 2023 14:47:58 +0800 Subject: [PATCH 318/856] Add a fix to address the recent TiDB data type issues (#720) Addresses https://github.com/sqlancer/sqlancer/issues/669. --- src/sqlancer/tidb/TiDBSchema.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 04bde5e33..609d1dbe7 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -238,6 +238,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { primitiveType = TiDBDataType.FLOATING; break; case "double": + case "double(8,6)": // workaround to address https://github.com/sqlancer/sqlancer/issues/669 size = 8; primitiveType = TiDBDataType.FLOATING; break; @@ -271,6 +272,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { break; case "date": case "datetime": + case "datetime(6)": // workaround to address https://github.com/sqlancer/sqlancer/issues/669 case "timestamp": case "time": case "year": From 0ba08b51de2ef94b698f5abce573269ad5719a50 Mon Sep 17 00:00:00 2001 From: Ming Cheng Date: Sun, 5 Feb 2023 21:09:13 -0500 Subject: [PATCH 319/856] Fix QuestDB execution bug (#721) --- src/sqlancer/questdb/QuestDBProvider.java | 2 +- src/sqlancer/questdb/QuestDBSchema.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/questdb/QuestDBProvider.java b/src/sqlancer/questdb/QuestDBProvider.java index d9932c8a3..79c1554be 100644 --- a/src/sqlancer/questdb/QuestDBProvider.java +++ b/src/sqlancer/questdb/QuestDBProvider.java @@ -106,7 +106,7 @@ public SQLConnection createDatabase(QuestDBGlobalState globalState) throws Excep } // TODO(anxing): maybe not hardcode here... String databaseName = "qdb"; - String tableName = "test"; + String tableName = "sqlancer_test"; String url = String.format("jdbc:postgresql://%s:%d/%s", host, port, databaseName); // use QuestDB default username & password for Postgres JDBC Properties properties = new Properties(); diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index 0b5563a31..61d27161a 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -177,7 +177,7 @@ public boolean isNullable() { public static class QuestDBTables extends AbstractTables { public static final Set RESERVED_TABLES = new HashSet<>( - Arrays.asList("sys.column_versions_purge_log", "telemetry_config", "telemetry")); + Arrays.asList("sys.column_versions_purge_log", "telemetry_config", "telemetry", "sys.telemetry_wal")); public QuestDBTables(List tables) { super(tables); From 504ab26125b7a1d67ed817ff65ffce536eac46ac Mon Sep 17 00:00:00 2001 From: Ming Cheng Date: Mon, 6 Feb 2023 00:50:33 -0500 Subject: [PATCH 320/856] Forbid multiple table selection from QuestDB (#722) * Forbid multiple table selection from QuestDB * Forbid multiple table selection from QuestDB * Format code * Remove unused field --- .../questdb/test/QuestDBQueryPartitioningBase.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index 45e3db9c4..5db302dbd 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -18,7 +18,6 @@ import sqlancer.questdb.QuestDBSchema; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBTable; -import sqlancer.questdb.QuestDBSchema.QuestDBTables; import sqlancer.questdb.ast.QuestDBExpression; import sqlancer.questdb.ast.QuestDBSelect; import sqlancer.questdb.gen.QuestDBExpressionGenerator; @@ -28,7 +27,7 @@ public class QuestDBQueryPartitioningBase implements TestOracle { QuestDBSchema s; - QuestDBTables targetTables; + QuestDBTable targetTable; // single table QuestDBExpressionGenerator gen; QuestDBSelect select; @@ -42,7 +41,7 @@ List> generateFetchColumns() { if (Randomly.getBoolean()) { columns.add(new ColumnReferenceNode<>(new QuestDBColumn("*", null, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + columns = Randomly.nonEmptySubset(targetTable.getColumns()).stream() .map(c -> new ColumnReferenceNode(c)) .collect(Collectors.toList()); } @@ -66,12 +65,15 @@ protected ExpressionGenerator> getGen() { @Override public void check() throws SQLException { s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new QuestDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + // Only return one table instead of multiple tables, which is regarded as illegal by QuestDB + // e.g. "SELECT * FROM t0, t1;" + targetTable = s.getRandomTable(); + gen = new QuestDBExpressionGenerator(state).setColumns(targetTable.getColumns()); initializeTernaryPredicateVariants(); select = new QuestDBSelect(); select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); + List tables = new ArrayList<>(); + tables.add(targetTable); List> tableList = tables.stream() .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); // Ignore JOINs for now From 40a1d26adaa0cc76050aa9380d271565862e4f95 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Wed, 8 Feb 2023 12:36:51 +0800 Subject: [PATCH 321/856] Update QGP preprint link (#724) --- docs/PAPERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/PAPERS.md b/docs/PAPERS.md index 45f57e887..6510930e1 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -53,7 +53,7 @@ This paper describes TLP, a metamorphic testing approach that can detect various ## Testing Database Engines via Query Plan Guidance -This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. +This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. A preprint is available [here](http://jinshengba.me/assets/pdf/qpg_icse23.pdf). ``` @inproceedings{Ba2023QPG, From aa0c0eccba4eefa75bfd518f608c9222c692c11d Mon Sep 17 00:00:00 2001 From: Ming Cheng Date: Sat, 11 Feb 2023 21:44:40 -0500 Subject: [PATCH 322/856] Add SYMBOL type support for QuestDB (#727) * add symbol support for QuestDB * add symbol support * fix: symbol type is null --- src/sqlancer/questdb/QuestDBSchema.java | 14 +++++++++-- src/sqlancer/questdb/ast/QuestDBConstant.java | 24 +++++++++++++++++++ .../gen/QuestDBExpressionGenerator.java | 4 ++++ .../test/QuestDBQueryPartitioningBase.java | 2 +- 4 files changed, 41 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index 61d27161a..e490d0cda 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -27,8 +27,7 @@ public enum QuestDBDataType { BOOLEAN, // CHAR, /* STRING, */ - INT, FLOAT, - /* SYMBOL, */ + INT, FLOAT, SYMBOL, // DATE, TIMESTAMP, /* GEOHASH, */ NULL; @@ -70,6 +69,9 @@ public QuestDBCompositeDataType(QuestDBDataType dataType, int size) { case BOOLEAN: isNullable = false; break; + case SYMBOL: + isNullable = true; + break; default: isNullable = true; } @@ -106,6 +108,9 @@ public static QuestDBCompositeDataType getRandomWithoutNull() { // case TIMESTAMP: size = 0; break; + case SYMBOL: + size = 0; + break; default: throw new AssertionError(type); } @@ -142,6 +147,8 @@ public String toString() { } case BOOLEAN: return Randomly.fromOptions("BOOLEAN"); + case SYMBOL: + return Randomly.fromOptions("SYMBOL"); // case TIMESTAMP: // return Randomly.fromOptions("TIMESTAMP"); // case DATE: @@ -233,6 +240,9 @@ private static QuestDBCompositeDataType getColumnType(String typeString) { primitiveType = QuestDBDataType.INT; size = 2; break; + case "SYMBOL": + primitiveType = QuestDBDataType.SYMBOL; + break; case "NULL": primitiveType = QuestDBDataType.NULL; break; diff --git a/src/sqlancer/questdb/ast/QuestDBConstant.java b/src/sqlancer/questdb/ast/QuestDBConstant.java index a59ce1053..f7e5ab0f8 100644 --- a/src/sqlancer/questdb/ast/QuestDBConstant.java +++ b/src/sqlancer/questdb/ast/QuestDBConstant.java @@ -47,6 +47,26 @@ public boolean getValue() { } } + public static class QuestDBSymbolConstant extends QuestDBConstant { + private final String value; + + public QuestDBSymbolConstant(String value) { + this.value = value; + } + + @Override + public String toString() { + if (value.equals("")) { + return "NULL"; + } + return "'" + value + "'"; + } + + public String getValue() { + return value; + } + } + public static Node createIntConstant(long val) { return new QuestDBIntConstant(val); } @@ -86,4 +106,8 @@ public static Node createNullConstant() { public static Node createFloatConstant(double val) { return new QuestDBDoubleConstant(val); } + + public static Node createSymbolConstant(String val) { + return new QuestDBSymbolConstant(val); + } } diff --git a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java index 8f9ea443a..73f57c055 100644 --- a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java @@ -5,6 +5,7 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.Randomly.StringGenerationStrategy; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; @@ -54,6 +55,9 @@ public Node generateConstant() { return QuestDBConstant.createBooleanConstant(Randomly.getBoolean()); case FLOAT: return QuestDBConstant.createFloatConstant(globalState.getRandomly().getDouble()); + case SYMBOL: + StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; + return QuestDBConstant.createSymbolConstant(strategy.getString(globalState.getRandomly())); // case CHAR: // case DATE: // case TIMESTAMP: diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index 5db302dbd..fa5283507 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -27,7 +27,7 @@ public class QuestDBQueryPartitioningBase implements TestOracle { QuestDBSchema s; - QuestDBTable targetTable; // single table + QuestDBTable targetTable; QuestDBExpressionGenerator gen; QuestDBSelect select; From 30948f34acc2354d6be18a70bdeeebff1e73fa48 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Feb 2023 18:18:54 +0800 Subject: [PATCH 323/856] Describe the expected errors concept --- CONTRIBUTING.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 08749f141..58b6af199 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,6 +21,10 @@ Please consider the following suggestions when creating a PR to contribute a ne * Add the DBMS' name to the [README.md](https://github.com/sqlancer/sqlancer/blob/master/README.md#supported-dbms) file. * It would be easier to review multiple smaller PRs, than one PR that contains the complete implementation. Consider contributing parts of your implementation as you work on their implementation. +## Expected Errors + +Most statements have an [ExpectedError](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/common/query/ExpectedErrors.java) object associated with them. This object essentially contains a list of errors, one of which the database system might return if it cannot successfully execute the statement. These errors are typically added through a trial-and-error process while considering various tradeoffs. For example, consider the [DuckDBInsertGenerator](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java#L38) class, whose expected errors are specified in [DuckDBErrors](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/DuckDBErrors.java#L90). When implementing such a generator, the list of expected errors might first be empty. When running the generator for the first time, you might receive an error such as "create unique index, table contains duplicate data", indicating that creating the index failed due to duplicate data. In principle, this error could be avoided by first checking whether the column contains any duplicate values. However, checking this would be expensive and error-prone (e.g., consider string similarity, which might depend on collations); thus, the obvious choice would be to add this string to the list of expected errors, and run the generator again to check for any other expected errors. In other cases, errors might be best addressed through improvements in the generators. For example, it is typically straightforward to generate syntactically-valid statements, which is why syntax errors should not be ignored. This approach is effective in uncovering internal errors; rather than ignoring them as an expected error, report them, and see [Unfixed Bugs](###unfixed-bugs) below. + ### Typed vs. Untyped Expression Generation Each DBMS implementation provides an expression generator used, for example, to generate expressions used in `WHERE` clauses. We found that DBMS can be roughly classified into "permissive" ones, which apply implicit type conversions when needed and "strict" ones, which provide only few implicit conversions and output an error when the type is unexpected. For example, consider the following test case: From 29c44513a2106f72b3610058368d3377135f8a8c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Feb 2023 18:30:08 +0800 Subject: [PATCH 324/856] Explain how to bail out while generating statements --- CONTRIBUTING.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 58b6af199..f2217b1f5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,9 +21,14 @@ Please consider the following suggestions when creating a PR to contribute a ne * Add the DBMS' name to the [README.md](https://github.com/sqlancer/sqlancer/blob/master/README.md#supported-dbms) file. * It would be easier to review multiple smaller PRs, than one PR that contains the complete implementation. Consider contributing parts of your implementation as you work on their implementation. -## Expected Errors +### Expected Errors + +Most statements have an [ExpectedError](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/common/query/ExpectedErrors.java) object associated with them. This object essentially contains a list of errors, one of which the database system might return if it cannot successfully execute the statement. These errors are typically added through a trial-and-error process while considering various tradeoffs. For example, consider the [DuckDBInsertGenerator](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java#L38) class, whose expected errors are specified in [DuckDBErrors](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/DuckDBErrors.java#L90). When implementing such a generator, the list of expected errors might first be empty. When running the generator for the first time, you might receive an error such as "create unique index, table contains duplicate data", indicating that creating the index failed due to duplicate data. In principle, this error could be avoided by first checking whether the column contains any duplicate values. However, checking this would be expensive and error-prone (e.g., consider string similarity, which might depend on collations); thus, the obvious choice would be to add this string to the list of expected errors, and run the generator again to check for any other expected errors. In other cases, errors might be best addressed through improvements in the generators. For example, it is typically straightforward to generate syntactically-valid statements, which is why syntax errors should not be ignored. This approach is effective in uncovering internal errors; rather than ignoring them as an expected error, report them, and see [Unfixed Bugs](#unfixed-bugs) below. + +### Bailing Out While Generating a Statement + +In some cases, it might be undesirable or even impossible to generate a specific statement type. For example, consider that SQLancer tries to execute a `DROP TABLE` statement (e.g., see [TiDBDropTableGenerator](https://github.com/sqlancer/sqlancer/blob/30948f34acc2354d6be18a70bdeeebff1e73fa48/src/sqlancer/tidb/gen/TiDBDropTableGenerator.java)), but the database contains only a single table. Dropping the table would result in all subsequent attempts to insert data or query it to fail. Thus, in such a case, it might be more efficient to "bail out" by abandoning the current attempt to generate the statement. This can be achieved by throwing a `IgnoreMeException`. Unlike for other exceptions, SQLancer silently continues execution rather than reporting this exception to the user. -Most statements have an [ExpectedError](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/common/query/ExpectedErrors.java) object associated with them. This object essentially contains a list of errors, one of which the database system might return if it cannot successfully execute the statement. These errors are typically added through a trial-and-error process while considering various tradeoffs. For example, consider the [DuckDBInsertGenerator](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java#L38) class, whose expected errors are specified in [DuckDBErrors](https://github.com/sqlancer/sqlancer/blob/aa0c0eccba4eefa75bfd518f608c9222c692c11d/src/sqlancer/duckdb/DuckDBErrors.java#L90). When implementing such a generator, the list of expected errors might first be empty. When running the generator for the first time, you might receive an error such as "create unique index, table contains duplicate data", indicating that creating the index failed due to duplicate data. In principle, this error could be avoided by first checking whether the column contains any duplicate values. However, checking this would be expensive and error-prone (e.g., consider string similarity, which might depend on collations); thus, the obvious choice would be to add this string to the list of expected errors, and run the generator again to check for any other expected errors. In other cases, errors might be best addressed through improvements in the generators. For example, it is typically straightforward to generate syntactically-valid statements, which is why syntax errors should not be ignored. This approach is effective in uncovering internal errors; rather than ignoring them as an expected error, report them, and see [Unfixed Bugs](###unfixed-bugs) below. ### Typed vs. Untyped Expression Generation From 0ec43d5974ebabb2faba86b28c17ed48514e610e Mon Sep 17 00:00:00 2001 From: Ming Cheng Date: Thu, 16 Feb 2023 22:18:08 -0500 Subject: [PATCH 325/856] Fix add index bug for QuestDB (#729) * add symbol support for QuestDB * add symbol support * fix: symbol type is null * Fix add index bug for QuestDB * Fix add index bug for QuestDB * Apply code formatter * Apply mvn verify for code * Compare data type directly instead of converting to string * Use equality operator --- src/sqlancer/common/schema/AbstractTable.java | 11 +++++++++++ src/sqlancer/questdb/QuestDBSchema.java | 2 +- .../questdb/gen/QuestDBAlterIndexGenerator.java | 10 +++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index c40f1a197..bf14abc1a 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -2,9 +2,11 @@ import java.util.Collections; import java.util.List; +import java.util.function.Predicate; import java.util.stream.Collectors; import sqlancer.GlobalState; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; public abstract class AbstractTable, I extends TableIndex, G extends GlobalState> @@ -60,6 +62,15 @@ public C getRandomColumn() { return Randomly.fromList(columns); } + public C getRandomColumnOrBailout(Predicate predicate) { + List relevantColumns = columns.stream().filter(predicate).collect(Collectors.toList()); + if (relevantColumns.isEmpty()) { + throw new IgnoreMeException(); + } + + return Randomly.fromList(relevantColumns); + } + public boolean hasIndexes() { return !indexes.isEmpty(); } diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index e490d0cda..8253ec82c 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -148,7 +148,7 @@ public String toString() { case BOOLEAN: return Randomly.fromOptions("BOOLEAN"); case SYMBOL: - return Randomly.fromOptions("SYMBOL"); + return "SYMBOL"; // case TIMESTAMP: // return Randomly.fromOptions("TIMESTAMP"); // case DATE: diff --git a/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java index 1e8242148..85103738f 100644 --- a/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBAlterIndexGenerator.java @@ -4,6 +4,8 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; +import sqlancer.questdb.QuestDBSchema.QuestDBColumn; +import sqlancer.questdb.QuestDBSchema.QuestDBDataType; import sqlancer.questdb.QuestDBSchema.QuestDBTable; public final class QuestDBAlterIndexGenerator { @@ -29,7 +31,13 @@ public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { sb.append(" "); sb.append("ALTER COLUMN "); - String columnName = table.getFreeColumnName(); + + // We should always choose column with SYMBOL type + QuestDBColumn columnWithSymbolType = table + .getRandomColumnOrBailout(c -> c.getType().getPrimitiveDataType() == QuestDBDataType.SYMBOL); + + String columnName = columnWithSymbolType.getName(); + sb.append(columnName); sb.append(" "); From b19a41e0dcf7cbd5fd84f66694743dc12323e2ef Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 2 Mar 2023 04:11:45 +0100 Subject: [PATCH 326/856] Add analyser flag (#738) --- src/sqlancer/clickhouse/ClickHouseOptions.java | 3 +++ src/sqlancer/clickhouse/ClickHouseProvider.java | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 0aafab84d..94297ad20 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -31,6 +31,9 @@ public class ClickHouseOptions implements DBMSSpecificOptions { TLPWhere { @Override diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 5c0c371c6..86f6fd392 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -143,7 +143,8 @@ public SQLConnection createDatabase(ClickHouseGlobalState globalState) throws SQ } con.close(); con = DriverManager.getConnection( - String.format("jdbc:clickhouse://%s:%d/%s?socket_timeout=300000", host, port, databaseName), + String.format("jdbc:clickhouse://%s:%d/%s?socket_timeout=300000%s", host, port, databaseName, + clickHouseOptions.enableAnalyzer ? "&allow_experimental_analyzer=1" : ""), globalState.getOptions().getUserName(), globalState.getOptions().getPassword()); return new SQLConnection(con); } From 5c5440f5aa4aad1be0c6a8c31dad19e4ce8273c8 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Sat, 4 Mar 2023 02:16:55 +0100 Subject: [PATCH 327/856] Add another error message for ambiguous column in ClickHouse (#741) * one more to skip * one more to skip --- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 97a5e300c..e824faef9 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -78,6 +78,8 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { // We may not count it as an issue, but it makes no sense to add more complex AST generation logic errors.add("MULTIPLE_EXPRESSIONS_FOR_ALIAS"); errors.add("AMBIGUOUS_IDENTIFIER"); // https://github.com/ClickHouse/ClickHouse/issues/45389 + errors.add("AMBIGUOUS_COLUMN_NAME"); // same https://github.com/ClickHouse/ClickHouse/issues/45389 + errors.add("Cannot parse number with multiple sign"); } } From 85bea1991954e0a39c0afbd485f8879a7b0a409a Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 6 Mar 2023 14:42:11 +0800 Subject: [PATCH 328/856] refactor: better code logic, try-with-resources with multiple resources (#750) * refactor: better code logic, try-with-resources with multiple resources * mvn formatter:format --- src/sqlancer/citus/CitusSchema.java | 56 +++++++++++++---------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index defda8804..560191c05 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -58,41 +58,35 @@ public Integer getColocationId() { public static CitusSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); - try { - List databaseTables = new ArrayList<>(); - try (Statement s = con.createStatement()) { - try (ResultSet rs = s.executeQuery( - "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { - while (rs.next()) { - String tableName = rs.getString("table_name"); - /* citus_tables is a helper view, we don't need to test with it so we let's ignore it */ - if (tableName.equals("citus_tables")) { - continue; - } - String distributionColumnName = rs.getString("dist_col_name"); - Integer colocationId = rs.getInt("colocationid"); - if (rs.wasNull()) { - colocationId = null; - } - PostgresTable t = schema.getDatabaseTable(tableName); - PostgresColumn distributionColumn = null; - if (t == null) { - continue; - } - if (distributionColumnName != null && !distributionColumnName.equals("")) { - distributionColumn = t.getColumns().stream() - .filter(c -> c.getName().equals(distributionColumnName)) - .collect(Collectors.toList()).get(0); - } - CitusTable tCitus = new CitusTable(t, distributionColumn, colocationId); - databaseTables.add(tCitus); - } + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement(); ResultSet rs = s.executeQuery( + "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + /* citus_tables is a helper view, we don't need to test with it so we let's ignore it */ + if (tableName.equals("citus_tables")) { + continue; } + String distributionColumnName = rs.getString("dist_col_name"); + Integer colocationId = rs.getInt("colocationid"); + if (rs.wasNull()) { + colocationId = null; + } + PostgresTable t = schema.getDatabaseTable(tableName); + PostgresColumn distributionColumn = null; + if (t == null) { + continue; + } + if (distributionColumnName != null && !distributionColumnName.equals("")) { + distributionColumn = t.getColumns().stream().filter(c -> c.getName().equals(distributionColumnName)) + .collect(Collectors.toList()).get(0); + } + CitusTable tCitus = new CitusTable(t, distributionColumn, colocationId); + databaseTables.add(tCitus); } - return new CitusSchema(databaseTables, databaseName); } catch (SQLIntegrityConstraintViolationException e) { throw new AssertionError(e); } + return new CitusSchema(databaseTables, databaseName); } - } From 466400139b4124be28093041fa4b7d4dd8c99885 Mon Sep 17 00:00:00 2001 From: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> Date: Tue, 7 Mar 2023 11:20:59 +0530 Subject: [PATCH 329/856] Refactor several classes (#752) * Refactor Several classes Signed-off-by: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> * mvn formatter:format Signed-off-by: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> * Checkstyle Error fix Signed-off-by: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> --------- Signed-off-by: Krishan Chavinda <90906063+krishan30@users.noreply.github.com> --- src/sqlancer/ProviderAdapter.java | 8 ++++---- src/sqlancer/Randomly.java | 15 +++++---------- src/sqlancer/SQLProviderAdapter.java | 2 +- src/sqlancer/StateToReproduce.java | 5 ++++- src/sqlancer/StatementExecutor.java | 2 +- src/sqlancer/common/DBMSCommon.java | 2 +- src/sqlancer/common/ast/BinaryNode.java | 2 +- src/sqlancer/common/ast/BinaryOperatorNode.java | 2 +- src/sqlancer/common/ast/FunctionNode.java | 2 +- src/sqlancer/common/ast/TernaryNode.java | 2 +- src/sqlancer/common/ast/UnaryNode.java | 2 +- src/sqlancer/common/ast/UnaryOperatorNode.java | 2 +- src/sqlancer/common/log/SQLLoggableFactory.java | 8 ++++---- src/sqlancer/common/oracle/NoRECBase.java | 2 +- .../common/oracle/PivotedQuerySynthesisBase.java | 2 +- src/sqlancer/common/schema/AbstractRowValue.java | 2 +- src/sqlancer/common/schema/AbstractSchema.java | 2 +- src/sqlancer/common/schema/AbstractTable.java | 4 ++-- 18 files changed, 32 insertions(+), 34 deletions(-) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 47b15ee39..ab5ca8e9a 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -26,7 +26,7 @@ public abstract class ProviderAdapter globalClass, Class optionClass) { + protected ProviderAdapter(Class globalClass, Class optionClass) { this.globalClass = globalClass; this.optionClass = optionClass; } @@ -85,7 +85,7 @@ protected TestOracle getTestOracle(G globalState) throws Exception { List> testOracleFactory = globalState.getDbmsSpecificOptions() .getTestOracleFactory(); boolean testOracleRequiresMoreThanZeroRows = testOracleFactory.stream() - .anyMatch(p -> p.requiresAllTablesToContainRows()); + .anyMatch(OracleFactory::requiresAllTablesToContainRows); boolean userRequiresMoreThanZeroRows = globalState.getOptions().testOnlyWithMoreThanZeroRows(); boolean checkZeroRows = testOracleRequiresMoreThanZeroRows || userRequiresMoreThanZeroRows; if (checkZeroRows && globalState.getSchema().containsTableWithZeroRows(globalState)) { @@ -98,7 +98,7 @@ protected TestOracle getTestOracle(G globalState) throws Exception { if (testOracleFactory.size() == 1) { return testOracleFactory.get(0).create(globalState); } else { - return new CompositeTestOracle(testOracleFactory.stream().map(o -> { + return new CompositeTestOracle<>(testOracleFactory.stream().map(o -> { try { return o.create(globalState); } catch (Exception e1) { @@ -170,7 +170,7 @@ private synchronized boolean mutateTables(G globalState) throws Exception { if (Randomly.getPercentage() < globalState.getOptions().getQPGProbability()) { selectedActionIndex = globalState.getRandomly().getInteger(0, weightedAverageReward.length); } else { - selectedActionIndex = DBMSCommon.getMaxIndexInDoubleArrary(weightedAverageReward); + selectedActionIndex = DBMSCommon.getMaxIndexInDoubleArray(weightedAverageReward); } int reward = 0; diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index fa317f1f0..ae34ba1b1 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -3,6 +3,7 @@ import java.math.BigDecimal; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Random; import java.util.function.Supplier; @@ -133,17 +134,12 @@ public static List subset(List columns) { public static List subset(int nr, @SuppressWarnings("unchecked") T... values) { List list = new ArrayList<>(); - for (T val : values) { - list.add(val); - } + Collections.addAll(list, values); return extractNrRandomColumns(list, nr); } public static List subset(@SuppressWarnings("unchecked") T... values) { - List list = new ArrayList<>(); - for (T val : values) { - list.add(val); - } + List list = new ArrayList<>(Arrays.asList(values)); return subset(list); } @@ -350,7 +346,6 @@ public long getNonZeroInteger() { do { value = getInteger(); } while (value == 0); - assert value != 0; addToCache(value); return value; } @@ -425,7 +420,7 @@ public long getLong(long left, long right) { } public BigDecimal getRandomBigDecimal() { - return new BigDecimal(getThreadRandom().get().nextDouble()); + return BigDecimal.valueOf(getThreadRandom().get().nextDouble()); } public long getPositiveIntegerNotNull() { @@ -494,7 +489,7 @@ private static long getNextLong(long lower, long upper) { if (lower == upper) { return lower; } - return (long) (getThreadRandom().get().longs(lower, upper).findFirst().getAsLong()); + return getThreadRandom().get().longs(lower, upper).findFirst().getAsLong(); } private static int getNextInt(int lower, int upper) { diff --git a/src/sqlancer/SQLProviderAdapter.java b/src/sqlancer/SQLProviderAdapter.java index 8de7523a3..efb4fab67 100644 --- a/src/sqlancer/SQLProviderAdapter.java +++ b/src/sqlancer/SQLProviderAdapter.java @@ -10,7 +10,7 @@ public abstract class SQLProviderAdapter>, O extends DBMSSpecificOptions>> extends ProviderAdapter { - public SQLProviderAdapter(Class globalClass, Class optionClass) { + protected SQLProviderAdapter(Class globalClass, Class optionClass) { super(globalClass, optionClass); } diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 7ac3e9ccf..33bfe3398 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -70,6 +70,9 @@ public List> getStatements() { return Collections.unmodifiableList(statements); } + /** + * @deprecated + */ @Deprecated public void commentStatements() { for (int i = 0; i < statements.size(); i++) { @@ -100,7 +103,7 @@ public class OracleRunReproductionState implements Closeable { private final List> statements = new ArrayList<>(); - public boolean success; + private boolean success; public OracleRunReproductionState() { StateToReproduce.this.localState = this; diff --git a/src/sqlancer/StatementExecutor.java b/src/sqlancer/StatementExecutor.java index 26265d136..4f8f48b8f 100644 --- a/src/sqlancer/StatementExecutor.java +++ b/src/sqlancer/StatementExecutor.java @@ -70,7 +70,7 @@ public void executeStatements() throws Exception { success = globalState.executeStatement(query); } while (nextAction.canBeRetried() && !success && nrTries++ < globalState.getOptions().getNrStatementRetryCount()); - } catch (IgnoreMeException e) { + } catch (IgnoreMeException ignored) { } if (query != null && query.couldAffectSchema()) { diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java index 122b34a83..f302d48ad 100644 --- a/src/sqlancer/common/DBMSCommon.java +++ b/src/sqlancer/common/DBMSCommon.java @@ -27,7 +27,7 @@ public static boolean matchesIndexName(String indexName) { return matcher.matches(); } - public static int getMaxIndexInDoubleArrary(double... doubleArray) { + public static int getMaxIndexInDoubleArray(double... doubleArray) { int maxIndex = 0; double maxValue = 0.0; for (int j = 0; j < doubleArray.length; j++) { diff --git a/src/sqlancer/common/ast/BinaryNode.java b/src/sqlancer/common/ast/BinaryNode.java index dec1854c4..90a56a078 100644 --- a/src/sqlancer/common/ast/BinaryNode.java +++ b/src/sqlancer/common/ast/BinaryNode.java @@ -7,7 +7,7 @@ public abstract class BinaryNode implements BinaryOperation { private final T left; private final T right; - public BinaryNode(T left, T right) { + protected BinaryNode(T left, T right) { this.left = left; this.right = right; } diff --git a/src/sqlancer/common/ast/BinaryOperatorNode.java b/src/sqlancer/common/ast/BinaryOperatorNode.java index 38d340a4b..586315059 100644 --- a/src/sqlancer/common/ast/BinaryOperatorNode.java +++ b/src/sqlancer/common/ast/BinaryOperatorNode.java @@ -10,7 +10,7 @@ public interface Operator { String getTextRepresentation(); } - public BinaryOperatorNode(T left, T right, O op) { + protected BinaryOperatorNode(T left, T right, O op) { super(left, right); this.op = op; } diff --git a/src/sqlancer/common/ast/FunctionNode.java b/src/sqlancer/common/ast/FunctionNode.java index a841d2956..ad795b998 100644 --- a/src/sqlancer/common/ast/FunctionNode.java +++ b/src/sqlancer/common/ast/FunctionNode.java @@ -7,7 +7,7 @@ public abstract class FunctionNode { protected F function; protected List args; - public FunctionNode(F function, List args) { + protected FunctionNode(F function, List args) { this.function = function; this.args = args; } diff --git a/src/sqlancer/common/ast/TernaryNode.java b/src/sqlancer/common/ast/TernaryNode.java index 146a46474..1a6605317 100644 --- a/src/sqlancer/common/ast/TernaryNode.java +++ b/src/sqlancer/common/ast/TernaryNode.java @@ -8,7 +8,7 @@ public abstract class TernaryNode implements BinaryOperation { private final T middle; private final T right; - public TernaryNode(T left, T middle, T right) { + protected TernaryNode(T left, T middle, T right) { this.left = left; this.middle = middle; this.right = right; diff --git a/src/sqlancer/common/ast/UnaryNode.java b/src/sqlancer/common/ast/UnaryNode.java index a6664c864..dea4220b9 100644 --- a/src/sqlancer/common/ast/UnaryNode.java +++ b/src/sqlancer/common/ast/UnaryNode.java @@ -6,7 +6,7 @@ public abstract class UnaryNode implements UnaryOperation { protected final T expr; - public UnaryNode(T expr) { + protected UnaryNode(T expr) { this.expr = expr; } diff --git a/src/sqlancer/common/ast/UnaryOperatorNode.java b/src/sqlancer/common/ast/UnaryOperatorNode.java index 979479c12..71556496c 100644 --- a/src/sqlancer/common/ast/UnaryOperatorNode.java +++ b/src/sqlancer/common/ast/UnaryOperatorNode.java @@ -6,7 +6,7 @@ public abstract class UnaryOperatorNode extends UnaryNode protected final O op; - public UnaryOperatorNode(T expr, O op) { + protected UnaryOperatorNode(T expr, O op) { super(expr); this.op = op; } diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java index 790d10709..7b7dfe4f1 100644 --- a/src/sqlancer/common/log/SQLLoggableFactory.java +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -37,10 +37,10 @@ public SQLQueryAdapter commentOutQuery(Query query) { @Override protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { StringBuilder sb = new StringBuilder(); - sb.append("-- Time: " + time + "\n"); - sb.append("-- Database: " + databaseName + "\n"); - sb.append("-- Database version: " + databaseVersion + "\n"); - sb.append("-- seed value: " + seedValue + "\n"); + sb.append("-- Time: ").append(time).append("\n"); + sb.append("-- Database: ").append(databaseName).append("\n"); + sb.append("-- Database version: ").append(databaseVersion).append("\n"); + sb.append("-- seed value: ").append(seedValue).append("\n"); return new LoggedString(sb.toString()); } diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java index c099b2592..2ac0dbb43 100644 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ b/src/sqlancer/common/oracle/NoRECBase.java @@ -16,7 +16,7 @@ public abstract class NoRECBase> implements TestO protected String optimizedQueryString; protected String unoptimizedQueryString; - public NoRECBase(S state) { + protected NoRECBase(S state) { this.state = state; this.con = state.getConnection(); this.logger = state.getLogger(); diff --git a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java index 6c551d18e..cde8a2e4b 100644 --- a/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java +++ b/src/sqlancer/common/oracle/PivotedQuerySynthesisBase.java @@ -29,7 +29,7 @@ public abstract class PivotedQuerySynthesisBase, protected final S globalState; protected R pivotRow; - public PivotedQuerySynthesisBase(S globalState) { + protected PivotedQuerySynthesisBase(S globalState) { this.globalState = globalState; } diff --git a/src/sqlancer/common/schema/AbstractRowValue.java b/src/sqlancer/common/schema/AbstractRowValue.java index c7f933c9b..2a979514d 100644 --- a/src/sqlancer/common/schema/AbstractRowValue.java +++ b/src/sqlancer/common/schema/AbstractRowValue.java @@ -63,7 +63,7 @@ public String asStringGroupedByTables() { sb.append("\n"); } AbstractTable t = tableList.get(j); - sb.append("-- " + t.getName() + "\n"); + sb.append("-- ").append(t.getName()).append("\n"); List columnsForTable = columnList.stream().filter(c -> c.getTable().equals(t)) .collect(Collectors.toList()); for (int i = 0; i < columnsForTable.size(); i++) { diff --git a/src/sqlancer/common/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java index 84a854f72..1c66363e7 100644 --- a/src/sqlancer/common/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -45,7 +45,7 @@ public A getRandomTable(Predicate predicate) { } public A getRandomTableOrBailout(Function f) { - List relevantTables = databaseTables.stream().filter(t -> f.apply(t)).collect(Collectors.toList()); + List relevantTables = databaseTables.stream().filter(f::apply).collect(Collectors.toList()); if (relevantTables.isEmpty()) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index bf14abc1a..4a57fa846 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -19,7 +19,7 @@ public abstract class AbstractTable, I exten private final boolean isView; protected long rowCount = NO_ROW_COUNT_AVAILABLE; - public AbstractTable(String name, List columns, List indexes, boolean isView) { + protected AbstractTable(String name, List columns, List indexes, boolean isView) { this.name = name; this.indexes = indexes; this.isView = isView; @@ -41,7 +41,7 @@ public String toString() { sb.append(getName()); sb.append("\n"); for (C c : columns) { - sb.append("\t" + c + "\n"); + sb.append("\t").append(c).append("\n"); } return sb.toString(); } From 3d5197701fcc7a927d2541922649ca14817d86e9 Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: Thu, 9 Mar 2023 11:14:14 +0500 Subject: [PATCH 330/856] Add regex error strings on encoding and infinity conversion to generalize skip of these errors. (#731) --- src/sqlancer/postgres/gen/PostgresCommon.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index a3e04d046..99bb7fa66 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; @@ -53,7 +54,7 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("invalid hexadecimal digit"); errors.add("invalid hexadecimal data: odd number of digits"); errors.add("zero raised to a negative power is undefined"); - errors.add("cannot convert infinity to numeric"); + errors.addRegex(Pattern.compile("cannot convert infinity to \\w+")); errors.add("division by zero"); errors.add("invalid input syntax for type money"); errors.add("invalid input syntax for type"); @@ -101,7 +102,20 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("requested character not valid for encoding"); // chr errors.add("requested length too large"); // repeat errors.add("invalid memory alloc request size"); // repeat - errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + + /* + * PostgreSQL support only a few conversion variants to ASCII: LATIN1, LATIN2, LATIN9 and WINDOWS1250. So, it is + * better to skip this error at all. + */ + errors.addRegex(Pattern.compile("encoding conversion from \\w+ to ASCII not supported")); + + /* + * In accordance with PostgreSQL code, commit 0ab1a2e, conversions to or from SQL_ASCII is meaningless. So + * disable errors on such an attempt. + */ + errors.addRegex(Pattern.compile("encoding conversion from SQL_ASCII to \\w+ not supported")); + errors.addRegex(Pattern.compile("encoding conversion from \\w+ to SQL_ASCII not supported")); + errors.add("negative substring length not allowed"); // substr errors.add("invalid mask length"); // set_masklen } From 2aea15a440fed7b0448cd7c6613be47a0e0455f4 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 9 Mar 2023 15:03:15 +0800 Subject: [PATCH 331/856] feat: add missing python naming convention test (#755) * feat: add missing python naming convention test * fix: fix working directory in GitHub Action --------- Co-authored-by: Manuel Rigger --- .github/workflows/main.yml | 21 +++++++--- src/check_names.py | 86 ++++++++++++++++++++++---------------- 2 files changed, 66 insertions(+), 41 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 329f108da..7c2ccf637 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -25,6 +25,12 @@ jobs: run: mvn -B verify -DskipTests=true - name: Misc Tests run: mvn -B '-Dtest=!sqlancer.dbms.**' test + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Naming Convention Tests + run: python src/check_names.py citus: name: DBMS Tests (Citus) @@ -72,7 +78,6 @@ jobs: - name: Run Tests run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test - clickhouse: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest @@ -101,6 +106,7 @@ jobs: docker rm clickhouse-server + cockroachdb: name: DBMS Tests (CockroachDB) runs-on: ubuntu-latest @@ -123,7 +129,7 @@ jobs: run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test - + cockroachdb-qpg: name: QPG Tests (CockroachDB) runs-on: ubuntu-latest @@ -173,6 +179,7 @@ jobs: run: | DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test + duckdb: name: DBMS Tests (DuckDB) runs-on: ubuntu-latest @@ -260,6 +267,7 @@ jobs: MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP + postgres: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest @@ -298,10 +306,11 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests - run: | + run: | mvn -Dtest=TestSQLitePQS test mvn -Dtest=TestSQLite3 test + sqlite-qpg: name: QPG Tests (SQLite) runs-on: ubuntu-latest @@ -317,9 +326,10 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests for QPG - run: | + run: | mvn -Dtest=TestSQLiteQPG test + tidb: name: DBMS Tests (TiDB) runs-on: ubuntu-latest @@ -365,7 +375,7 @@ jobs: run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test - + yugabyte: name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest @@ -421,7 +431,6 @@ jobs: - name: Shortly run DuckDB run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - java15: name: Java 15 EA Compatibility (DuckDB) runs-on: ubuntu-latest diff --git a/src/check_names.py b/src/check_names.py index 31d6853a4..9208cc00e 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -1,38 +1,54 @@ import os +import sys +from typing import List -def get_java_files(directory): - java_files = [] - for root, dirs, files in os.walk(directory): - for f in files: - if f.endswith('.java'): - java_files.append(f) - return java_files - -def verify_prefix(prefix, files): - if len(files) == 0: - print(prefix + ' directory does not contain any files!') - exit(-1) - for f in files: - if not f.startswith(prefix): - print('The class name of ' + f + ' does not start with ' + prefix) - exit(-1) - -verify_prefix('ArangoDB', get_java_files("sqlancer/arangodb/")) -verify_prefix('Citus', get_java_files("sqlancer/citus/")) -verify_prefix('ClickHouse', get_java_files("sqlancer/clickhouse/")) -verify_prefix('CnosDB', get_java_files("sqlancer/cnosdb/")) -verify_prefix('CockroachDB', get_java_files("sqlancer/cockroachdb/")) -verify_prefix('Cosmos', get_java_files("sqlancer/cosmos/")) -verify_prefix('Databend', get_java_files("sqlancer/databend/")) -verify_prefix('DuckDB', get_java_files("sqlancer/duckdb")) -verify_prefix('H2', get_java_files("sqlancer/h2")) -verify_prefix('HSQLDB', get_java_files("sqlancer/hsqldb")) -verify_prefix('MariaDB', get_java_files("sqlancer/mariadb/")) -verify_prefix('MySQL', get_java_files("sqlancer/mysql/")) -verify_prefix('OceanBase', get_java_files("sqlancer/oceanbase/")) -verify_prefix('Postgres', get_java_files("sqlancer/postgres/")) -verify_prefix('QuestDB', get_java_files("sqlancer/questdb/")) -verify_prefix('SQLite3', get_java_files("sqlancer/sqlite3/")) -verify_prefix('TiDB', get_java_files("sqlancer/tidb/")) -verify_prefix('Y', get_java_files("sqlancer/yugabyte/")) # has both YCQL and YSQL prefixes +def get_java_files(directory_path: str) -> List[str]: + java_files: List[str] = [] + for root, dirs, files in os.walk(directory_path): + for f in files: + if f.endswith('.java'): + java_files.append(f) + return java_files + + +def verify_one_db(prefix: str, files: List[str]): + print('checking database, name: {0}, files: {1}'.format(prefix, files)) + if len(files) == 0: + print(prefix + ' directory does not contain any files!', file=sys.stderr) + exit(-1) + for f in files: + if not f.startswith(prefix): + print('The class name of ' + f + ' does not start with ' + prefix, file=sys.stderr) + exit(-1) + print('checking database pass: ', prefix) + + +def verify_all_dbs(name_to_files: dict[str:List[str]]): + for db_name, files in name_to_files.items(): + verify_one_db(db_name, files) + + +if __name__ == '__main__': + cwd = os.getcwd() + print("Current working directory: {0}".format(cwd)) + name_to_files: dict[str:List[str]] = dict() + name_to_files["ArangoDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "arangodb")) + name_to_files["Citus"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "citus")) + name_to_files["ClickHouse"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "clickhouse")) + name_to_files["CnosDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cnosdb")) + name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) + name_to_files["Cosmos"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cosmos")) + name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) + name_to_files["DuckDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "duckdb")) + name_to_files["H2"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "h2")) + name_to_files["HSQLDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "hsqldb")) + name_to_files["MariaDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mariadb")) + name_to_files["MySQL"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mysql")) + name_to_files["OceanBase"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "oceanbase")) + name_to_files["Postgres"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "postgres")) + name_to_files["QuestDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "questdb")) + name_to_files["SQLite3"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "sqlite3")) + name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) + name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes + verify_all_dbs(name_to_files) From 7e77b2a75b8dccf6a7676014cb547637176c5202 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Fri, 10 Mar 2023 19:23:03 +0800 Subject: [PATCH 332/856] feat: add support for TimescaleDB (#748) * feat: add TimeScaleDB, initial basic structure of bugs, global state and schema * feat: add TimeScaleDB, check_names.py * feat: add Common, Error, Options, Provider. modify Schema * style: run mvn formatter * fix: error: The method getTimeScaleDBErrors() from the TimeScaleDBProvider is never used locally * fix: TestUsageNamingConvention.testNonEmptyDescription:23 TimeScaleDB should be in lowercase! * fix: change access right * refactor: rename TimeScaleDB to TimescaleDB * style: run mvn formatter * fix: remove expected errors timesaledb --- src/sqlancer/timescaledb/TimescaleDBBugs.java | 7 +++ .../timescaledb/TimescaleDBGlobalState.java | 12 +++++ .../timescaledb/TimescaleDBOptions.java | 27 ++++++++++ .../timescaledb/TimescaleDBProvider.java | 30 +++++++++++ .../timescaledb/TimescaleDBSchema.java | 52 +++++++++++++++++++ .../timescaledb/gen/TimescaleDBCommon.java | 13 +++++ 6 files changed, 141 insertions(+) create mode 100644 src/sqlancer/timescaledb/TimescaleDBBugs.java create mode 100644 src/sqlancer/timescaledb/TimescaleDBGlobalState.java create mode 100644 src/sqlancer/timescaledb/TimescaleDBOptions.java create mode 100644 src/sqlancer/timescaledb/TimescaleDBProvider.java create mode 100644 src/sqlancer/timescaledb/TimescaleDBSchema.java create mode 100644 src/sqlancer/timescaledb/gen/TimescaleDBCommon.java diff --git a/src/sqlancer/timescaledb/TimescaleDBBugs.java b/src/sqlancer/timescaledb/TimescaleDBBugs.java new file mode 100644 index 000000000..079dfee74 --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBBugs.java @@ -0,0 +1,7 @@ +package sqlancer.timescaledb; + +public final class TimescaleDBBugs { + private TimescaleDBBugs() { + + } +} diff --git a/src/sqlancer/timescaledb/TimescaleDBGlobalState.java b/src/sqlancer/timescaledb/TimescaleDBGlobalState.java new file mode 100644 index 000000000..519001b90 --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBGlobalState.java @@ -0,0 +1,12 @@ +package sqlancer.timescaledb; + +import java.sql.SQLException; + +import sqlancer.postgres.PostgresGlobalState; + +public class TimescaleDBGlobalState extends PostgresGlobalState { + @Override + public TimescaleDBSchema readSchema() throws SQLException { + return TimescaleDBSchema.fromConnection(getConnection(), getDatabaseName()); + } +} diff --git a/src/sqlancer/timescaledb/TimescaleDBOptions.java b/src/sqlancer/timescaledb/TimescaleDBOptions.java new file mode 100644 index 000000000..03b479662 --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBOptions.java @@ -0,0 +1,27 @@ +package sqlancer.timescaledb; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; + +public class TimescaleDBOptions extends PostgresOptions { + @Parameter(names = "--timescaledboracle", description = "Specifies which test oracle should be used for TimeScaleDB extension to PostgreSQL") + public List timescaleDBOracle = Arrays.asList(TimescaleDBOracleFactory.PQS); + + public enum TimescaleDBOracleFactory implements OracleFactory { + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + }, + } +} diff --git a/src/sqlancer/timescaledb/TimescaleDBProvider.java b/src/sqlancer/timescaledb/TimescaleDBProvider.java new file mode 100644 index 000000000..756669128 --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBProvider.java @@ -0,0 +1,30 @@ +package sqlancer.timescaledb; + +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.PostgresProvider; +import sqlancer.timescaledb.gen.TimescaleDBCommon; + +@AutoService(DatabaseProvider.class) +public class TimescaleDBProvider extends PostgresProvider { + @SuppressWarnings("unchecked") + public TimescaleDBProvider() { + super((Class) (Object) TimescaleDBGlobalState.class, + (Class) (Object) TimescaleDBOptions.class); + } + + @Override + public String getDBMSName() { + return "timescaledb"; + } + + public static ExpectedErrors getTimescaleDBErrors() { + ExpectedErrors errors = new ExpectedErrors(); + TimescaleDBCommon.addTimescaleDBErrors(errors); + return errors; + } +} diff --git a/src/sqlancer/timescaledb/TimescaleDBSchema.java b/src/sqlancer/timescaledb/TimescaleDBSchema.java new file mode 100644 index 000000000..0c09d579c --- /dev/null +++ b/src/sqlancer/timescaledb/TimescaleDBSchema.java @@ -0,0 +1,52 @@ +package sqlancer.timescaledb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.SQLConnection; +import sqlancer.postgres.PostgresSchema; + +public class TimescaleDBSchema extends PostgresSchema { + + public TimescaleDBSchema(List databaseTables, String databaseName) { + super(new ArrayList<>(databaseTables), databaseName); + } + + public static class TimescaleDBTable extends PostgresTable { + public TimescaleDBTable(String tableName, List columns, List indexes, + TableType tableType, List statistics, boolean isView, boolean isInsertable) { + super(tableName, columns, indexes, tableType, statistics, isView, isInsertable); + } + + public TimescaleDBTable(PostgresTable table) { + super(table.getName(), table.getColumns(), table.getIndexes(), table.getTableType(), table.getStatistics(), + table.isView(), table.isInsertable()); + } + } + + public static TimescaleDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); + List databaseTables = new ArrayList<>(); + try (Statement s = con.createStatement(); + ResultSet rs = s.executeQuery("SELECT table_name FROM information_schema.tables")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + + PostgresTable t = schema.getDatabaseTable(tableName); + if (t == null) { + continue; + } + + TimescaleDBTable table = new TimescaleDBTable(t); + databaseTables.add(table); + } + } catch (SQLIntegrityConstraintViolationException e) { + throw new AssertionError(e); + } + return new TimescaleDBSchema(databaseTables, databaseName); + } +} diff --git a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java new file mode 100644 index 000000000..1a13b9cb4 --- /dev/null +++ b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java @@ -0,0 +1,13 @@ +package sqlancer.timescaledb.gen; + +import sqlancer.common.query.ExpectedErrors; + +public final class TimescaleDBCommon { + private TimescaleDBCommon() { + + } + + public static void addTimescaleDBErrors(ExpectedErrors errors) { + + } +} From 354b95369b618f74a2e3b920ef7f75d963fcebd4 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 19 Mar 2023 18:54:42 +0800 Subject: [PATCH 333/856] Disable QPG tests in general Action (#766) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7c2ccf637..3abbac71a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,7 +24,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**' test + run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - name: Setup Python uses: actions/setup-python@v4 with: From 07db8b1be8447d10dc0a5f04fdb7e8d77e2890d4 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 20 Mar 2023 13:09:30 +0800 Subject: [PATCH 334/856] fix: ahere to max number of queries in QPG (#767) --- src/sqlancer/ProviderAdapter.java | 2 +- src/sqlancer/common/query/SQLQueryAdapter.java | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index ab5ca8e9a..1972557d6 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -125,7 +125,7 @@ public void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws E while (executedQueryCount < globalState.getOptions().getNrQueries()) { int numOfNoNewQueryPlans = 0; TestOracle oracle = getTestOracle(globalState); - while (true) { + while (executedQueryCount < globalState.getOptions().getNrQueries()) { try (OracleRunReproductionState localState = globalState.getState().createLocalState()) { assert localState != null; try { diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index ab9124230..22eef268a 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -92,6 +92,8 @@ public String getUnterminatedQueryString() { Main.nrUnsuccessfulActions.addAndGet(1); checkException(e); return false; + } finally { + s.close(); } } From bc0fa8e7a0634dc637f1fd94a694618a9578c115 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Apr 2023 08:25:42 +0200 Subject: [PATCH 335/856] Add Materialize support (#773) * Copy postgres -> materialize as basis * Comment out all Postgres actions which don't work in Materialize * Rename Postgres to Materialize * Use table name in index to make sure we don't try recreating same index names * Fix formatting * Remove commented out original Pg code * Revert index name change * Add Materialize to CI * Prevent creating many indexes on Materialize * More reasonable defaults * Materialize: Fix QPG * Add Materialize to README * Fix issue running Main() method twice --- .github/workflows/main.yml | 50 ++ README.md | 3 +- configs/spotbugs-exclude.xml | 5 + src/check_names.py | 1 + .../MaterializeCompoundDataType.java | 46 ++ .../MaterializeExpectedValueVisitor.java | 162 ++++++ .../materialize/MaterializeGlobalState.java | 292 ++++++++++ .../materialize/MaterializeOptions.java | 93 ++++ .../materialize/MaterializeProvider.java | 331 +++++++++++ .../materialize/MaterializeSchema.java | 330 +++++++++++ .../MaterializeToStringVisitor.java | 335 ++++++++++++ .../materialize/MaterializeVisitor.java | 127 +++++ .../materialize/ast/MaterializeAggregate.java | 58 ++ .../materialize/ast/MaterializeAlias.java | 35 ++ .../ast/MaterializeBetweenOperation.java | 66 +++ .../MaterializeBinaryArithmeticOperation.java | 103 ++++ .../ast/MaterializeBinaryBitOperation.java | 46 ++ .../MaterializeBinaryComparisonOperation.java | 121 ++++ .../MaterializeBinaryLogicalOperation.java | 89 +++ .../ast/MaterializeBinaryRangeOperation.java | 74 +++ .../ast/MaterializeCastOperation.java | 45 ++ .../ast/MaterializeColumnValue.java | 34 ++ .../ast/MaterializeConcatOperation.java | 37 ++ .../materialize/ast/MaterializeConstant.java | 516 +++++++++++++++++ .../ast/MaterializeExpression.java | 14 + .../materialize/ast/MaterializeFunction.java | 256 +++++++++ .../MaterializeFunctionWithUnknownResult.java | 148 +++++ .../ast/MaterializeInOperation.java | 66 +++ .../materialize/ast/MaterializeJoin.java | 50 ++ .../ast/MaterializeLikeOperation.java | 38 ++ .../ast/MaterializeOrderByTerm.java | 42 ++ .../MaterializePOSIXRegularExpression.java | 65 +++ .../ast/MaterializePostfixOperation.java | 151 +++++ .../ast/MaterializePostfixText.java | 37 ++ .../ast/MaterializePrefixOperation.java | 119 ++++ .../materialize/ast/MaterializeSelect.java | 135 +++++ .../materialize/ast/MaterializeSimilarTo.java | 40 ++ .../materialize/gen/MaterializeCommon.java | 351 ++++++++++++ .../gen/MaterializeDeleteGenerator.java | 38 ++ .../gen/MaterializeDropIndexGenerator.java | 53 ++ .../gen/MaterializeExpressionGenerator.java | 517 ++++++++++++++++++ .../gen/MaterializeIndexGenerator.java | 80 +++ .../gen/MaterializeInsertGenerator.java | 106 ++++ .../gen/MaterializeRandomQueryGenerator.java | 64 +++ .../gen/MaterializeTableGenerator.java | 127 +++++ .../gen/MaterializeUpdateGenerator.java | 78 +++ .../gen/MaterializeViewGenerator.java | 73 +++ .../oracle/MaterializeNoRECOracle.java | 176 ++++++ ...aterializePivotedQuerySynthesisOracle.java | 151 +++++ .../tlp/MaterializeTLPAggregateOracle.java | 187 +++++++ .../oracle/tlp/MaterializeTLPBase.java | 123 +++++ .../tlp/MaterializeTLPHavingOracle.java | 83 +++ .../oracle/tlp/MaterializeTLPWhereOracle.java | 52 ++ test/sqlancer/dbms/TestMaterialize.java | 24 + test/sqlancer/dbms/TestMaterializePQS.java | 25 + .../qpg/materialize/TestMaterializeQPG.java | 23 + .../materialize/TestMaterializeQueryPlan.java | 49 ++ 57 files changed, 6539 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/materialize/MaterializeCompoundDataType.java create mode 100644 src/sqlancer/materialize/MaterializeExpectedValueVisitor.java create mode 100644 src/sqlancer/materialize/MaterializeGlobalState.java create mode 100644 src/sqlancer/materialize/MaterializeOptions.java create mode 100644 src/sqlancer/materialize/MaterializeProvider.java create mode 100644 src/sqlancer/materialize/MaterializeSchema.java create mode 100644 src/sqlancer/materialize/MaterializeToStringVisitor.java create mode 100644 src/sqlancer/materialize/MaterializeVisitor.java create mode 100644 src/sqlancer/materialize/ast/MaterializeAggregate.java create mode 100644 src/sqlancer/materialize/ast/MaterializeAlias.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBetweenOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeCastOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeColumnValue.java create mode 100644 src/sqlancer/materialize/ast/MaterializeConcatOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeConstant.java create mode 100644 src/sqlancer/materialize/ast/MaterializeExpression.java create mode 100644 src/sqlancer/materialize/ast/MaterializeFunction.java create mode 100644 src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java create mode 100644 src/sqlancer/materialize/ast/MaterializeInOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeJoin.java create mode 100644 src/sqlancer/materialize/ast/MaterializeLikeOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeOrderByTerm.java create mode 100644 src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java create mode 100644 src/sqlancer/materialize/ast/MaterializePostfixOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializePostfixText.java create mode 100644 src/sqlancer/materialize/ast/MaterializePrefixOperation.java create mode 100644 src/sqlancer/materialize/ast/MaterializeSelect.java create mode 100644 src/sqlancer/materialize/ast/MaterializeSimilarTo.java create mode 100644 src/sqlancer/materialize/gen/MaterializeCommon.java create mode 100644 src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeIndexGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeInsertGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeTableGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java create mode 100644 src/sqlancer/materialize/gen/MaterializeViewGenerator.java create mode 100644 src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java create mode 100644 src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java create mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java create mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java create mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java create mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java create mode 100644 test/sqlancer/dbms/TestMaterialize.java create mode 100644 test/sqlancer/dbms/TestMaterializePQS.java create mode 100644 test/sqlancer/qpg/materialize/TestMaterializeQPG.java create mode 100644 test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3abbac71a..1892285b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -239,6 +239,56 @@ jobs: - name: Run Tests run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test + materialize: + name: DBMS Tests (Materialize) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Materialize + run: | + docker pull materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest + sleep 5 + # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 + psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterialize + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializePQS + + materialize-qpg: + name: QPG Tests (Materialize) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up Materialize + run: | + docker pull materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest + sleep 5 + # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 + psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQPG + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQueryPlan + mysql: name: DBMS Tests (MySQL) runs-on: ubuntu-18.04 diff --git a/README.md b/README.md index 346209908..59f69cce2 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ If you launch SQLancer without parameters, available options and commands are di | Approach | Description | |----------|-------------| | Random Generation | Random generation is the default test case generation approach in SQLancer. First, random tables are generated. Then queries are randomly generated based on the schemas of the tables. | -| Query Plan Guidance (QPG) | QPG is a test case generation method guided by query plan coverage. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. This approach is enabled by option `--qpg-enable` and now supports TLP and NoREC oracles for SQLite, CockroachDB, and TiDB. | +| Query Plan Guidance (QPG) | QPG is a test case generation method guided by query plan coverage. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. This approach is enabled by option `--qpg-enable` and now supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. | Please find the `.bib` entries [here](docs/PAPERS.md). @@ -69,6 +69,7 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | Databend | Working | Typed | | | QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | | CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | +| Materialize |Working | Typed | | # Using SQLancer diff --git a/configs/spotbugs-exclude.xml b/configs/spotbugs-exclude.xml index 5cc1baf73..7fa4de560 100644 --- a/configs/spotbugs-exclude.xml +++ b/configs/spotbugs-exclude.xml @@ -7,6 +7,11 @@ + + + + + diff --git a/src/check_names.py b/src/check_names.py index 9208cc00e..6dd4f1a72 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -44,6 +44,7 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["H2"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "h2")) name_to_files["HSQLDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "hsqldb")) name_to_files["MariaDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mariadb")) + name_to_files["Materialize"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "materialize")) name_to_files["MySQL"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mysql")) name_to_files["OceanBase"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "oceanbase")) name_to_files["Postgres"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "postgres")) diff --git a/src/sqlancer/materialize/MaterializeCompoundDataType.java b/src/sqlancer/materialize/MaterializeCompoundDataType.java new file mode 100644 index 000000000..a4bddf7ca --- /dev/null +++ b/src/sqlancer/materialize/MaterializeCompoundDataType.java @@ -0,0 +1,46 @@ +package sqlancer.materialize; + +import java.util.Optional; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public final class MaterializeCompoundDataType { + + private final MaterializeDataType dataType; + private final MaterializeCompoundDataType elemType; + private final Integer size; + + private MaterializeCompoundDataType(MaterializeDataType dataType, MaterializeCompoundDataType elemType, + Integer size) { + this.dataType = dataType; + this.elemType = elemType; + this.size = size; + } + + public MaterializeDataType getDataType() { + return dataType; + } + + public MaterializeCompoundDataType getElemType() { + if (elemType == null) { + throw new AssertionError(); + } + return elemType; + } + + public Optional getSize() { + if (size == null) { + return Optional.empty(); + } else { + return Optional.of(size); + } + } + + public static MaterializeCompoundDataType create(MaterializeDataType type, int size) { + return new MaterializeCompoundDataType(type, null, size); + } + + public static MaterializeCompoundDataType create(MaterializeDataType type) { + return new MaterializeCompoundDataType(type, null, null); + } +} diff --git a/src/sqlancer/materialize/MaterializeExpectedValueVisitor.java b/src/sqlancer/materialize/MaterializeExpectedValueVisitor.java new file mode 100644 index 000000000..caa801268 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeExpectedValueVisitor.java @@ -0,0 +1,162 @@ +package sqlancer.materialize; + +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSimilarTo; + +public final class MaterializeExpectedValueVisitor implements MaterializeVisitor { + + private final StringBuilder sb = new StringBuilder(); + private static final int NR_TABS = 0; + + private void print(MaterializeExpression expr) { + MaterializeToStringVisitor v = new MaterializeToStringVisitor(); + v.visit(expr); + for (int i = 0; i < NR_TABS; i++) { + sb.append("\t"); + } + sb.append(v.get()); + sb.append(" -- "); + sb.append(expr.getExpectedValue()); + sb.append("\n"); + } + + @Override + public void visit(MaterializeConstant constant) { + print(constant); + } + + @Override + public void visit(MaterializePostfixOperation op) { + print(op); + visit(op.getExpression()); + } + + public String get() { + return sb.toString(); + } + + @Override + public void visit(MaterializeColumnValue c) { + print(c); + } + + @Override + public void visit(MaterializePrefixOperation op) { + print(op); + visit(op.getExpression()); + } + + @Override + public void visit(MaterializeSelect op) { + visit(op.getWhereClause()); + } + + @Override + public void visit(MaterializeOrderByTerm op) { + + } + + @Override + public void visit(MaterializeFunction f) { + print(f); + for (int i = 0; i < f.getArguments().length; i++) { + visit(f.getArguments()[i]); + } + } + + @Override + public void visit(MaterializeCastOperation cast) { + print(cast); + visit(cast.getExpression()); + } + + @Override + public void visit(MaterializeBetweenOperation op) { + print(op); + visit(op.getExpr()); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(MaterializeInOperation op) { + print(op); + visit(op.getExpr()); + for (MaterializeExpression right : op.getListElements()) { + visit(right); + } + } + + @Override + public void visit(MaterializePostfixText op) { + print(op); + visit(op.getExpr()); + } + + @Override + public void visit(MaterializeAggregate op) { + print(op); + for (MaterializeExpression expr : op.getArgs()) { + visit(expr); + } + } + + @Override + public void visit(MaterializeSimilarTo op) { + print(op); + visit(op.getString()); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + } + + @Override + public void visit(MaterializePOSIXRegularExpression op) { + print(op); + visit(op.getString()); + visit(op.getRegex()); + } + + @Override + public void visit(MaterializeFromTable from) { + print(from); + } + + @Override + public void visit(MaterializeSubquery subquery) { + print(subquery); + } + + @Override + public void visit(MaterializeBinaryLogicalOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + @Override + public void visit(MaterializeLikeOperation op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + +} diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java new file mode 100644 index 000000000..77cbe5d14 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -0,0 +1,292 @@ +package sqlancer.materialize; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; + +public class MaterializeGlobalState extends SQLGlobalState { + + public static final char IMMUTABLE = 'i'; + public static final char STABLE = 's'; + public static final char VOLATILE = 'v'; + + private List operators = Collections.emptyList(); + private List collates = Collections.emptyList(); + private List opClasses = Collections.emptyList(); + private List tableAccessMethods = Collections.emptyList(); + // store and allow filtering by function volatility classifications + private final Map functionsAndTypes = new HashMap<>(); + private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + + @Override + public void setConnection(SQLConnection con) { + super.setConnection(con); + try { + this.opClasses = getOpclasses(); + this.operators = getOperators(getConnection()); + this.collates = getCollnames(getConnection()); + this.tableAccessMethods = getTableAccessMethods(getConnection()); + } catch (SQLException e) { + throw new AssertionError(e); + } + } + + private List getCollnames(SQLConnection con) throws SQLException { + List collNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("SELECT collname FROM pg_collation WHERE collname LIKE '%utf8' or collname = 'C';")) { + while (rs.next()) { + collNames.add(rs.getString(1)); + } + } + } + return collNames; + } + + private List getOpclasses() throws SQLException { + List opClasses = new ArrayList<>(); + // select opcname FROM pg_opclass; + // ERROR: unknown catalog item 'pg_opclass' + opClasses.add("array_ops"); + opClasses.add("array_ops"); + opClasses.add("bit_ops"); + opClasses.add("bool_ops"); + opClasses.add("bpchar_ops"); + opClasses.add("bpchar_ops"); + opClasses.add("bytea_ops"); + opClasses.add("char_ops"); + opClasses.add("char_ops"); + opClasses.add("cidr_ops"); + opClasses.add("cidr_ops"); + opClasses.add("date_ops"); + opClasses.add("date_ops"); + opClasses.add("float4_ops"); + opClasses.add("float4_ops"); + opClasses.add("float8_ops"); + opClasses.add("float8_ops"); + opClasses.add("inet_ops"); + opClasses.add("inet_ops"); + opClasses.add("inet_ops"); + opClasses.add("inet_ops"); + opClasses.add("int2_ops"); + opClasses.add("int2_ops"); + opClasses.add("int4_ops"); + opClasses.add("int4_ops"); + opClasses.add("int8_ops"); + opClasses.add("int8_ops"); + opClasses.add("interval_ops"); + opClasses.add("interval_ops"); + opClasses.add("macaddr_ops"); + opClasses.add("macaddr_ops"); + opClasses.add("macaddr8_ops"); + opClasses.add("macaddr8_ops"); + opClasses.add("name_ops"); + opClasses.add("name_ops"); + opClasses.add("numeric_ops"); + opClasses.add("numeric_ops"); + opClasses.add("oid_ops"); + opClasses.add("oid_ops"); + opClasses.add("oidvector_ops"); + opClasses.add("oidvector_ops"); + opClasses.add("record_ops"); + opClasses.add("record_image_ops"); + opClasses.add("text_ops"); + opClasses.add("text_ops"); + opClasses.add("time_ops"); + opClasses.add("time_ops"); + opClasses.add("timestamptz_ops"); + opClasses.add("timestamptz_ops"); + opClasses.add("timetz_ops"); + opClasses.add("timetz_ops"); + opClasses.add("varbit_ops"); + opClasses.add("varchar_ops"); + opClasses.add("varchar_ops"); + opClasses.add("timestamp_ops"); + opClasses.add("timestamp_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("money_ops"); + opClasses.add("bool_ops"); + opClasses.add("bytea_ops"); + opClasses.add("tid_ops"); + opClasses.add("xid_ops"); + opClasses.add("cid_ops"); + opClasses.add("tid_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("aclitem_ops"); + opClasses.add("box_ops"); + opClasses.add("point_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("money_ops"); + opClasses.add("bool_ops"); + opClasses.add("bytea_ops"); + opClasses.add("tid_ops"); + opClasses.add("xid_ops"); + opClasses.add("cid_ops"); + opClasses.add("tid_ops"); + opClasses.add("text_pattern_ops"); + opClasses.add("varchar_pattern_ops"); + opClasses.add("bpchar_pattern_ops"); + opClasses.add("aclitem_ops"); + opClasses.add("box_ops"); + opClasses.add("point_ops"); + opClasses.add("poly_ops"); + opClasses.add("circle_ops"); + opClasses.add("array_ops"); + opClasses.add("uuid_ops"); + opClasses.add("uuid_ops"); + opClasses.add("pg_lsn_ops"); + opClasses.add("pg_lsn_ops"); + opClasses.add("enum_ops"); + opClasses.add("enum_ops"); + opClasses.add("tsvector_ops"); + opClasses.add("tsvector_ops"); + opClasses.add("tsvector_ops"); + opClasses.add("tsquery_ops"); + opClasses.add("tsquery_ops"); + opClasses.add("range_ops"); + opClasses.add("range_ops"); + opClasses.add("range_ops"); + opClasses.add("range_ops"); + opClasses.add("box_ops"); + opClasses.add("quad_point_ops"); + opClasses.add("kd_point_ops"); + opClasses.add("text_ops"); + opClasses.add("poly_ops"); + opClasses.add("jsonb_ops"); + opClasses.add("jsonb_ops"); + opClasses.add("jsonb_ops"); + opClasses.add("jsonb_path_ops"); + opClasses.add("bytea_minmax_ops"); + opClasses.add("char_minmax_ops"); + opClasses.add("name_minmax_ops"); + opClasses.add("int8_minmax_ops"); + opClasses.add("int2_minmax_ops"); + opClasses.add("int4_minmax_ops"); + opClasses.add("text_minmax_ops"); + opClasses.add("oid_minmax_ops"); + opClasses.add("tid_minmax_ops"); + opClasses.add("float4_minmax_ops"); + opClasses.add("float8_minmax_ops"); + opClasses.add("macaddr_minmax_ops"); + opClasses.add("macaddr8_minmax_ops"); + opClasses.add("inet_minmax_ops"); + opClasses.add("inet_inclusion_ops"); + opClasses.add("bpchar_minmax_ops"); + opClasses.add("time_minmax_ops"); + opClasses.add("date_minmax_ops"); + opClasses.add("timestamp_minmax_ops"); + opClasses.add("timestamptz_minmax_ops"); + opClasses.add("interval_minmax_ops"); + opClasses.add("timetz_minmax_ops"); + opClasses.add("bit_minmax_ops"); + opClasses.add("varbit_minmax_ops"); + opClasses.add("numeric_minmax_ops"); + opClasses.add("uuid_minmax_ops"); + opClasses.add("range_inclusion_ops"); + opClasses.add("pg_lsn_minmax_ops"); + opClasses.add("box_inclusion_ops"); + return opClasses; + } + + private List getOperators(SQLConnection con) throws SQLException { + List operators = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SELECT oprname FROM pg_operator;")) { + while (rs.next()) { + operators.add(rs.getString(1)); + } + } + } + return operators; + } + + private List getTableAccessMethods(SQLConnection con) throws SQLException { + List tableAccessMethods = new ArrayList<>(); + try (Statement s = con.createStatement()) { + /* + * pg_am includes both index and table access methods so we need to filter with amtype = 't' + */ + try (ResultSet rs = s.executeQuery("SELECT amname FROM pg_am WHERE amtype = 't';")) { + while (rs.next()) { + tableAccessMethods.add(rs.getString(1)); + } + } + } + return tableAccessMethods; + } + + public List getOperators() { + return operators; + } + + public String getRandomOperator() { + return Randomly.fromList(operators); + } + + public List getCollates() { + return collates; + } + + public String getRandomCollate() { + return Randomly.fromList(collates); + } + + public List getOpClasses() { + return opClasses; + } + + public String getRandomOpclass() { + return Randomly.fromList(opClasses); + } + + public List getTableAccessMethods() { + return tableAccessMethods; + } + + public String getRandomTableAccessMethod() { + return Randomly.fromList(tableAccessMethods); + } + + @Override + public MaterializeSchema readSchema() throws SQLException { + return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + } + + public void addFunctionAndType(String functionName, Character functionType) { + this.functionsAndTypes.put(functionName, functionType); + } + + public Map getFunctionsAndTypes() { + return this.functionsAndTypes; + } + + public void setAllowedFunctionTypes(List types) { + this.allowedFunctionTypes = types; + } + + public void setDefaultAllowedFunctionTypes() { + this.allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + } + + public List getAllowedFunctionTypes() { + return this.allowedFunctionTypes; + } + +} diff --git a/src/sqlancer/materialize/MaterializeOptions.java b/src/sqlancer/materialize/MaterializeOptions.java new file mode 100644 index 000000000..23191e3a9 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeOptions.java @@ -0,0 +1,93 @@ +package sqlancer.materialize; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.materialize.MaterializeOptions.MaterializeOracleFactory; +import sqlancer.materialize.oracle.MaterializeNoRECOracle; +import sqlancer.materialize.oracle.MaterializePivotedQuerySynthesisOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPAggregateOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPHavingOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPWhereOracle; + +@Parameters(separators = "=", commandDescription = "Materialize (default port: " + MaterializeOptions.DEFAULT_PORT + + ", default host: " + MaterializeOptions.DEFAULT_HOST + ", default user: " + MaterializeOptions.DEFAULT_USER + + ")") +public class MaterializeOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final String DEFAULT_USER = "materialize"; + public static final int DEFAULT_PORT = 6875; + + @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) + public boolean allowBulkInsert; + + @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for Materialize") + public List oracle = Arrays.asList(MaterializeOracleFactory.QUERY_PARTITIONING); + + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) + public boolean testCollations = true; + + @Parameter(names = "--set-max-tables-mvs", description = "Specifies whether to set the maximum number of tables and materialized views intiially", arity = 1) + public boolean setMaxTablesMVs; + + @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the Materialize server", arity = 1) + public String connectionURL = String.format("postgresql://%s@%s:%d/test", MaterializeOptions.DEFAULT_USER, + MaterializeOptions.DEFAULT_HOST, MaterializeOptions.DEFAULT_PORT); + + @Parameter(names = "--extensions", description = "Specifies a comma-separated list of extension names to be created in each test database", arity = 1) + public String extensions = ""; + + public enum MaterializeOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializeNoRECOracle(globalState); + } + }, + PQS { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + HAVING { + + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializeTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); + oracles.add(new MaterializeTLPWhereOracle(globalState)); + oracles.add(new MaterializeTLPHavingOracle(globalState)); + oracles.add(new MaterializeTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + + } + + @Override + public List getTestOracleFactory() { + return oracle; + } + +} diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java new file mode 100644 index 000000000..dc802d2aa --- /dev/null +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -0,0 +1,331 @@ +package sqlancer.materialize; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.net.URI; +import java.net.URISyntaxException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Arrays; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.materialize.MaterializeOptions.MaterializeOracleFactory; +import sqlancer.materialize.gen.MaterializeDeleteGenerator; +import sqlancer.materialize.gen.MaterializeDropIndexGenerator; +import sqlancer.materialize.gen.MaterializeIndexGenerator; +import sqlancer.materialize.gen.MaterializeInsertGenerator; +import sqlancer.materialize.gen.MaterializeTableGenerator; +import sqlancer.materialize.gen.MaterializeUpdateGenerator; +import sqlancer.materialize.gen.MaterializeViewGenerator; + +// EXISTS +// IN +@AutoService(DatabaseProvider.class) +public class MaterializeProvider extends SQLProviderAdapter { + + /** + * Generate only data types and expressions that are understood by PQS. + */ + public static boolean generateOnlyKnown; + + protected String entryURL; + protected String username; + protected String password; + protected String entryPath; + protected String host; + protected int port; + protected String testURL; + protected String databaseName; + protected String createDatabaseCommand; + protected String extensionsList; + + public MaterializeProvider() { + super(MaterializeGlobalState.class, MaterializeOptions.class); + } + + protected MaterializeProvider(Class globalClass, Class optionClass) { + super(globalClass, optionClass); + } + + public enum Action implements AbstractAction { + DELETE(MaterializeDeleteGenerator::create), // + DROP_INDEX(MaterializeDropIndexGenerator::create), // + INSERT(MaterializeInsertGenerator::insert), // + UPDATE(MaterializeUpdateGenerator::create), // + CREATE_INDEX(MaterializeIndexGenerator::generate), // + CREATE_VIEW(MaterializeViewGenerator::create); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(MaterializeGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + protected static int mapActions(MaterializeGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + int nrPerformed; + switch (a) { + case CREATE_INDEX: + nrPerformed = r.getInteger(0, 3); + break; + case DROP_INDEX: + nrPerformed = r.getInteger(0, 5); + break; + case DELETE: + nrPerformed = r.getInteger(0, 5); + break; + case CREATE_VIEW: + nrPerformed = r.getInteger(0, 2); + break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; + case INSERT: + nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + break; + default: + throw new AssertionError(a); + } + return nrPerformed; + + } + + @Override + public void generateDatabase(MaterializeGlobalState globalState) throws Exception { + readFunctions(globalState); + createTables(globalState, Randomly.fromOptions(4, 5, 6)); + prepareTables(globalState); + + extensionsList = globalState.getDbmsSpecificOptions().extensions; + if (!extensionsList.isEmpty()) { + String[] extensionNames = extensionsList.split(","); + + /* + * To avoid of a test interference with an extension objects, create them in a separate schema. Of course, + * they must be truly relocatable. + */ + globalState.executeStatement(new SQLQueryAdapter("CREATE SCHEMA extensions;", true)); + for (int i = 0; i < extensionNames.length; i++) { + globalState.executeStatement(new SQLQueryAdapter( + "CREATE EXTENSION " + extensionNames[i] + " WITH SCHEMA extensions;", true)); + } + } + } + + @Override + public SQLConnection createDatabase(MaterializeGlobalState globalState) throws SQLException { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == MaterializeOracleFactory.PQS)) { + generateOnlyKnown = true; + } + + username = globalState.getOptions().getUserName(); + password = globalState.getOptions().getPassword(); + host = globalState.getOptions().getHost(); + port = globalState.getOptions().getPort(); + entryPath = "/test"; + entryURL = globalState.getDbmsSpecificOptions().connectionURL; + // trim URL to exclude "jdbc:" + if (entryURL.startsWith("jdbc:")) { + entryURL = entryURL.substring(5); + } + String entryDatabaseName = entryPath.substring(1); + databaseName = globalState.getDatabaseName(); + + try { + URI uri = new URI(entryURL); + String userInfoURI = uri.getUserInfo(); + String pathURI = uri.getPath(); + if (userInfoURI != null) { + // username and password specified in URL take precedence + if (userInfoURI.contains(":")) { + String[] userInfo = userInfoURI.split(":", 2); + username = userInfo[0]; + password = userInfo[1]; + } else { + username = userInfoURI; + password = null; + } + int userInfoIndex = entryURL.indexOf(userInfoURI); + String preUserInfo = entryURL.substring(0, userInfoIndex); + String postUserInfo = entryURL.substring(userInfoIndex + userInfoURI.length() + 1); + entryURL = preUserInfo + postUserInfo; + } + if (pathURI != null) { + entryPath = pathURI; + } + if (host == null) { + host = uri.getHost(); + } + if (port == MainOptions.NO_SET_PORT) { + port = uri.getPort(); + } + entryURL = String.format("%s://%s:%d/%s", uri.getScheme(), host, port, entryDatabaseName); + } catch (URISyntaxException e) { + throw new AssertionError(e); + } + Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); + globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + createDatabaseCommand = getCreateDatabaseCommand(globalState); + globalState.getState().logStatement(createDatabaseCommand); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute(createDatabaseCommand); + } + con.close(); + if (globalState.getDbmsSpecificOptions().setMaxTablesMVs) { + Connection conMzSystem = DriverManager.getConnection("jdbc:postgresql://localhost:6877/materialize", + "mz_system", "materialize"); + try (Statement s = conMzSystem.createStatement()) { + s.execute("ALTER SYSTEM SET max_tables TO 1000"); + } + try (Statement s = conMzSystem.createStatement()) { + s.execute("ALTER SYSTEM SET max_materialized_views TO 1000"); + } + conMzSystem.close(); + } + int databaseIndex = entryURL.indexOf(entryDatabaseName); + String preDatabaseName = entryURL.substring(0, databaseIndex); + String postDatabaseName = entryURL.substring(databaseIndex + entryDatabaseName.length()); + testURL = preDatabaseName + databaseName + postDatabaseName; + globalState.getState().logStatement(String.format("\\c %s;", databaseName)); + + con = DriverManager.getConnection("jdbc:" + testURL, username, password); + return new SQLConnection(con); + } + + protected void readFunctions(MaterializeGlobalState globalState) throws SQLException { + // ERROR: column "provolatile" does not exist + SQLQueryAdapter query = new SQLQueryAdapter("SELECT proname, 1 FROM pg_proc;"); + SQLancerResultSet rs = query.executeAndGet(globalState); + while (rs.next()) { + String functionName = rs.getString(1); + Character functionType = rs.getString(2).charAt(0); + globalState.addFunctionAndType(functionName, functionType); + } + } + + protected void createTables(MaterializeGlobalState globalState, int numTables) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < numTables) { + try { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = MaterializeTableGenerator.generate(tableName, globalState.getSchema(), + generateOnlyKnown, globalState); + globalState.executeStatement(createTable); + } catch (IgnoreMeException e) { + + } + } + } + + protected void prepareTables(MaterializeGlobalState globalState) throws Exception { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + MaterializeProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + globalState.executeStatement(new SQLQueryAdapter("COMMIT", true)); + globalState.executeStatement(new SQLQueryAdapter("SET SESSION statement_timeout = 5000;\n")); + } + + private String getCreateDatabaseCommand(MaterializeGlobalState state) { + StringBuilder sb = new StringBuilder(); + sb.append("CREATE DATABASE " + databaseName + " "); + if (Randomly.getBoolean() && ((MaterializeOptions) state.getDbmsSpecificOptions()).testCollations) { + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } + } + } + return sb.toString(); + } + + @Override + public String getDBMSName() { + return "materialize"; + } + + @Override + public String getQueryPlan(String selectStr, MaterializeGlobalState globalState) throws Exception { + String queryPlan = ""; + String explainQuery = "EXPLAIN OPTIMIZED PLAN FOR " + selectStr; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery); + boolean afterProjection = false; // Remove the concrete expression after each Projection operator + SQLancerResultSet rs = q.executeAndGet(globalState); + if (rs != null) { + while (rs.next()) { + String line; + BufferedReader bufReader = new BufferedReader(new StringReader(rs.getString(1))); + while ((line = bufReader.readLine()) != null) { + String targetQueryPlan = line.trim() + ";"; // Unify format + if (targetQueryPlan.startsWith("Explained Query:")) { + continue; + } + if (afterProjection) { + afterProjection = false; + continue; + } + if (targetQueryPlan.startsWith("Project")) { + afterProjection = true; + } + // Remove all concrete expressions by keywords + if (targetQueryPlan.contains(">") || targetQueryPlan.contains("<") || targetQueryPlan.contains("=") + || targetQueryPlan.contains("*") || targetQueryPlan.contains("+") + || targetQueryPlan.contains("'")) { + continue; + } + queryPlan += targetQueryPlan; + } + } + } + + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[Action.values().length]; + } + + @Override + protected void executeMutator(int index, MaterializeGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } +} diff --git a/src/sqlancer/materialize/MaterializeSchema.java b/src/sqlancer/materialize/MaterializeSchema.java new file mode 100644 index 000000000..1762c5cb9 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeSchema.java @@ -0,0 +1,330 @@ +package sqlancer.materialize; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.SQLIntegrityConstraintViolationException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.postgresql.util.PSQLException; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTable.TableType; +import sqlancer.materialize.ast.MaterializeConstant; + +public class MaterializeSchema extends AbstractSchema { + + private final String databaseName; + private final List indexNames; + + public List getIndexNames() { + return indexNames; + } + + public enum MaterializeDataType { + INT, BOOLEAN, TEXT, DECIMAL, FLOAT, REAL, BIT; + + public static MaterializeDataType getRandomType() { + List dataTypes = new ArrayList<>(Arrays.asList(values())); + if (MaterializeProvider.generateOnlyKnown) { + dataTypes.remove(MaterializeDataType.DECIMAL); + dataTypes.remove(MaterializeDataType.FLOAT); + dataTypes.remove(MaterializeDataType.REAL); + dataTypes.remove(MaterializeDataType.BIT); + } + return Randomly.fromList(dataTypes); + } + } + + public static class MaterializeColumn extends AbstractTableColumn { + + public MaterializeColumn(String name, MaterializeDataType columnType) { + super(name, null, columnType); + } + + public static MaterializeColumn createDummy(String name) { + return new MaterializeColumn(name, MaterializeDataType.INT); + } + + } + + public static class MaterializeTables extends AbstractTables { + + public MaterializeTables(List tables) { + super(tables); + } + + public MaterializeRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String randomRow = String.format("SELECT %s FROM %s LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet randomRowValues = s.executeQuery(randomRow); + if (!randomRowValues.next()) { + throw new AssertionError("could not find random row! " + randomRow + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + MaterializeColumn column = getColumns().get(i); + int columnIndex = randomRowValues.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + MaterializeConstant constant; + if (randomRowValues.getString(columnIndex) == null) { + constant = MaterializeConstant.createNullConstant(); + } else { + switch (column.getType()) { + case INT: + constant = MaterializeConstant.createIntConstant(randomRowValues.getLong(columnIndex)); + break; + case BOOLEAN: + constant = MaterializeConstant + .createBooleanConstant(randomRowValues.getBoolean(columnIndex)); + break; + case TEXT: + constant = MaterializeConstant.createTextConstant(randomRowValues.getString(columnIndex)); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !randomRowValues.next(); + return new MaterializeRowValue(this, values); + } catch (PSQLException e) { + throw new IgnoreMeException(); + } + + } + + } + + public static MaterializeDataType getColumnType(String typeString) { + switch (typeString) { + case "smallint": + case "integer": + case "bigint": + return MaterializeDataType.INT; + case "boolean": + return MaterializeDataType.BOOLEAN; + case "text": + case "character": + case "character varying": + case "name": + case "regclass": + return MaterializeDataType.TEXT; + case "numeric": + return MaterializeDataType.DECIMAL; + case "double precision": + return MaterializeDataType.FLOAT; + case "real": + return MaterializeDataType.REAL; + case "bit": + return MaterializeDataType.BIT; + default: + throw new AssertionError(typeString); + } + } + + public static class MaterializeRowValue + extends AbstractRowValue { + + protected MaterializeRowValue(MaterializeTables tables, Map values) { + super(tables, values); + } + + } + + public static class MaterializeTable + extends AbstractRelationalTable { + + public enum TableType { + STANDARD, TEMPORARY + } + + private final TableType tableType; + private final List statistics; + private final boolean isInsertable; + + public MaterializeTable(String tableName, List columns, List indexes, + TableType tableType, List statistics, boolean isView, + boolean isInsertable) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + } + + public List getStatistics() { + return statistics; + } + + public TableType getTableType() { + return tableType; + } + + public boolean isInsertable() { + return isInsertable; + } + + } + + public static final class MaterializeStatisticsObject { + private final String name; + + public MaterializeStatisticsObject(String name) { + this.name = name; + } + + public String getName() { + return name; + } + } + + public static final class MaterializeIndex extends TableIndex { + + private MaterializeIndex(String indexName) { + super(indexName); + } + + public static MaterializeIndex create(String indexName) { + return new MaterializeIndex(indexName); + } + + @Override + public String getIndexName() { + if (super.getIndexName().contentEquals("PRIMARY")) { + return "`PRIMARY`"; + } else { + return super.getIndexName(); + } + } + + } + + public static MaterializeSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + try { + List databaseTables = new ArrayList<>(); + List indexNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + // ERROR: column "is_insertable_into" does not exist + try (ResultSet rs = s.executeQuery( + "SELECT table_name, table_schema, table_type FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { + while (rs.next()) { + String tableName = rs.getString("table_name"); + String tableTypeSchema = rs.getString("table_schema"); + boolean isInsertable = true; + String type = rs.getString("table_type"); + boolean isView = type.equals("VIEW") || type.equals("MATERIALIZED VIEW"); + if (isView) { + isInsertable = false; + } + MaterializeTable.TableType tableType = getTableType(tableTypeSchema); + List databaseColumns = getTableColumns(con, tableName); + List indexes = getIndexes(con, tableName); + List statistics = getStatistics(con); + MaterializeTable t = new MaterializeTable(tableName, databaseColumns, indexes, tableType, + statistics, isView, isInsertable); + for (MaterializeColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + } + } + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT mz_indexes.name, mz_databases.name FROM mz_indexes JOIN mz_relations ON mz_indexes.on_id = mz_relations.id JOIN mz_schemas ON mz_relations.schema_id = mz_schemas.id JOIN mz_databases ON mz_schemas.database_id = mz_databases.id WHERE mz_databases.name = '%s';", + databaseName))) { + while (rs.next()) { + String name = rs.getString(1); + indexNames.add(name); + } + } + } + return new MaterializeSchema(databaseTables, databaseName, indexNames); + } catch (SQLIntegrityConstraintViolationException e) { + throw new AssertionError(e); + } + } + + protected static List getStatistics(SQLConnection con) throws SQLException { + return new ArrayList<>(); + } + + protected static MaterializeTable.TableType getTableType(String tableTypeStr) throws AssertionError { + MaterializeTable.TableType tableType; + if (tableTypeStr.contentEquals("public")) { + tableType = TableType.STANDARD; + } else if (tableTypeStr.startsWith("pg_temp")) { + tableType = TableType.TEMPORARY; + } else { + throw new AssertionError(tableTypeStr); + } + return tableType; + } + + protected static List getIndexes(SQLConnection con, String tableName) throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String + // org.postgresql.util.PSQLException: ERROR: unknown catalog item 'pg_indexes' + .format("SELECT c.relname as indexname FROM pg_catalog.pg_class c LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_catalog.pg_index i ON i.indexrelid = c.oid LEFT JOIN pg_catalog.pg_class c2 ON i.indrelid = c2.oid WHERE c.relkind IN ('i','I','') AND n.nspname <> 'pg_catalog' AND n.nspname !~ '^pg_toast' AND n.nspname <> 'information_schema' AND c2.relname = '%s' AND pg_catalog.pg_table_is_visible(c.oid) ORDER BY indexname;", + tableName))) { + while (rs.next()) { + String indexName = rs.getString("indexname"); + if (DBMSCommon.matchesIndexName(indexName)) { + indexes.add(MaterializeIndex.create(indexName)); + } + } + } + } + return indexes; + } + + protected static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s + .executeQuery("select column_name, data_type from INFORMATION_SCHEMA.COLUMNS where table_name = '" + + tableName + "' ORDER BY column_name")) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + MaterializeColumn c = new MaterializeColumn(columnName, getColumnType(dataType)); + columns.add(c); + } + } + } + return columns; + } + + public MaterializeSchema(List databaseTables, String databaseName, List indexNames) { + super(databaseTables); + this.databaseName = databaseName; + this.indexNames = indexNames; + } + + public MaterializeTables getRandomTableNonEmptyTables() { + return new MaterializeTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public String getDatabaseName() { + return databaseName; + } + +} diff --git a/src/sqlancer/materialize/MaterializeToStringVisitor.java b/src/sqlancer/materialize/MaterializeToStringVisitor.java new file mode 100644 index 000000000..6a3f06445 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeToStringVisitor.java @@ -0,0 +1,335 @@ +package sqlancer.materialize; + +import java.util.Optional; + +import sqlancer.Randomly; +import sqlancer.common.visitor.BinaryOperation; +import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSimilarTo; + +public final class MaterializeToStringVisitor extends ToStringVisitor + implements MaterializeVisitor { + + @Override + public void visitSpecific(MaterializeExpression expr) { + MaterializeVisitor.super.visit(expr); + } + + @Override + public void visit(MaterializeConstant constant) { + sb.append(constant.getTextRepresentation()); + } + + @Override + public String get() { + return sb.toString(); + } + + @Override + public void visit(MaterializePostfixOperation op) { + sb.append("("); + visit(op.getExpression()); + sb.append(")"); + sb.append(" "); + sb.append(op.getOperatorTextRepresentation()); + } + + @Override + public void visit(MaterializeColumnValue c) { + sb.append(c.getColumn().getFullQualifiedName()); + } + + @Override + public void visit(MaterializePrefixOperation op) { + sb.append(op.getTextRepresentation()); + sb.append(" ("); + visit(op.getExpression()); + sb.append(")"); + } + + @Override + public void visit(MaterializeFromTable from) { + sb.append(from.getTable().getName()); + } + + @Override + public void visit(MaterializeSubquery subquery) { + sb.append("("); + visit(subquery.getSelect()); + sb.append(") AS "); + sb.append(subquery.getName()); + } + + @Override + public void visit(MaterializeSelect s) { + sb.append("SELECT "); + switch (s.getSelectOption()) { + case DISTINCT: + sb.append("DISTINCT "); + if (s.getDistinctOnClause() != null) { + sb.append("ON ("); + visit(s.getDistinctOnClause()); + sb.append(") "); + } + break; + case ALL: + sb.append(Randomly.fromOptions("ALL ", "")); + break; + default: + throw new AssertionError(); + } + visit(s.getFetchColumns()); + sb.append(" FROM "); + visit(s.getFromList()); + + for (MaterializeJoin j : s.getJoinClauses()) { + sb.append(" "); + switch (j.getType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } + sb.append("JOIN"); + break; + case LEFT: + sb.append("LEFT OUTER JOIN"); + break; + case RIGHT: + sb.append("RIGHT OUTER JOIN"); + break; + case FULL: + sb.append("FULL OUTER JOIN"); + break; + case CROSS: + sb.append("CROSS JOIN"); + break; + default: + throw new AssertionError(j.getType()); + } + sb.append(" "); + visit(j.getTableReference()); + if (j.getType() != MaterializeJoinType.CROSS) { + sb.append(" ON "); + visit(j.getOnClause()); + } + } + + if (s.getWhereClause() != null) { + sb.append(" WHERE "); + visit(s.getWhereClause()); + } + if (s.getGroupByExpressions().size() > 0) { + sb.append(" GROUP BY "); + visit(s.getGroupByExpressions()); + } + if (s.getHavingClause() != null) { + sb.append(" HAVING "); + visit(s.getHavingClause()); + + } + if (!s.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(s.getOrderByExpressions()); + } + if (s.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(s.getLimitClause()); + } + + if (s.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(s.getOffsetClause()); + } + } + + @Override + public void visit(MaterializeOrderByTerm op) { + visit(op.getExpr()); + sb.append(" "); + sb.append(op.getOrder()); + } + + @Override + public void visit(MaterializeFunction f) { + sb.append(f.getFunctionName()); + sb.append("("); + int i = 0; + for (MaterializeExpression arg : f.getArguments()) { + if (i++ != 0) { + sb.append(", "); + } + visit(arg); + } + sb.append(")"); + } + + @Override + public void visit(MaterializeCastOperation cast) { + if (cast.getCompoundType().getDataType() == MaterializeDataType.BOOLEAN) { + sb.append("("); + MaterializeExpression expr = cast.getExpression(); + visit(expr); + if (expr.getExpressionType() == MaterializeDataType.TEXT) { + sb.append(" != '')"); + } else if (expr.getExpressionType() == MaterializeDataType.BOOLEAN) { + sb.append(" != FALSE)"); + } else { + sb.append(" != 0)"); + } + } else if (Randomly.getBoolean()) { + if (cast.getCompoundType().getDataType() == MaterializeDataType.REAL + || cast.getCompoundType().getDataType() == MaterializeDataType.FLOAT) { + sb.append("CAST(CAST("); + visit(cast.getExpression()); + sb.append(" AS INT) AS "); + appendType(cast); + sb.append(")"); + } else { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + appendType(cast); + sb.append(")"); + } + } else { + if (cast.getCompoundType().getDataType() == MaterializeDataType.REAL + || cast.getCompoundType().getDataType() == MaterializeDataType.FLOAT) { + sb.append("("); + visit(cast.getExpression()); + sb.append(")::INT::"); + appendType(cast); + } else { + sb.append("("); + visit(cast.getExpression()); + sb.append(")::"); + appendType(cast); + } + } + } + + private void appendType(MaterializeCastOperation cast) { + MaterializeCompoundDataType compoundType = cast.getCompoundType(); + switch (compoundType.getDataType()) { + case BOOLEAN: + sb.append("BOOLEAN"); + break; + case INT: // TODO support also other int types + sb.append("INT"); + break; + case TEXT: + // TODO: append TEXT, CHAR + sb.append(Randomly.fromOptions("VARCHAR")); + break; + case REAL: + sb.append("FLOAT"); + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + sb.append("REAL"); + break; + case BIT: + sb.append("INT"); + break; + default: + throw new AssertionError(cast.getType()); + } + Optional size = compoundType.getSize(); + if (size.isPresent()) { + sb.append("("); + sb.append(size.get()); + sb.append(")"); + } + } + + @Override + public void visit(MaterializeBetweenOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(") BETWEEN "); + sb.append("("); + visit(op.getLeft()); + sb.append(") AND ("); + visit(op.getRight()); + sb.append(")"); + } + + @Override + public void visit(MaterializeInOperation op) { + sb.append("("); + visit(op.getExpr()); + sb.append(")"); + if (!op.isTrue()) { + sb.append(" NOT"); + } + sb.append(" IN ("); + visit(op.getListElements()); + sb.append(")"); + } + + @Override + public void visit(MaterializePostfixText op) { + visit(op.getExpr()); + sb.append(op.getText()); + } + + @Override + public void visit(MaterializeAggregate op) { + sb.append(op.getFunction()); + sb.append("("); + visit(op.getArgs()); + sb.append(")"); + } + + @Override + public void visit(MaterializeSimilarTo op) { + sb.append("("); + visit(op.getString()); + sb.append(" SIMILAR TO "); + visit(op.getSimilarTo()); + if (op.getEscapeCharacter() != null) { + visit(op.getEscapeCharacter()); + } + sb.append(")"); + } + + @Override + public void visit(MaterializePOSIXRegularExpression op) { + visit(op.getString()); + sb.append(op.getOp().getStringRepresentation()); + visit(op.getRegex()); + } + + @Override + public void visit(MaterializeBinaryLogicalOperation op) { + super.visit((BinaryOperation) op); + } + + @Override + public void visit(MaterializeLikeOperation op) { + super.visit((BinaryOperation) op); + } + +} diff --git a/src/sqlancer/materialize/MaterializeVisitor.java b/src/sqlancer/materialize/MaterializeVisitor.java new file mode 100644 index 000000000..40ed4e3f2 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeVisitor.java @@ -0,0 +1,127 @@ +package sqlancer.materialize; + +import java.util.List; + +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSimilarTo; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public interface MaterializeVisitor { + + void visit(MaterializeConstant constant); + + void visit(MaterializePostfixOperation op); + + void visit(MaterializeColumnValue c); + + void visit(MaterializePrefixOperation op); + + void visit(MaterializeSelect op); + + void visit(MaterializeOrderByTerm op); + + void visit(MaterializeFunction f); + + void visit(MaterializeCastOperation cast); + + void visit(MaterializeBetweenOperation op); + + void visit(MaterializeInOperation op); + + void visit(MaterializePostfixText op); + + void visit(MaterializeAggregate op); + + void visit(MaterializeSimilarTo op); + + void visit(MaterializePOSIXRegularExpression op); + + void visit(MaterializeFromTable from); + + void visit(MaterializeSubquery subquery); + + void visit(MaterializeBinaryLogicalOperation op); + + void visit(MaterializeLikeOperation op); + + default void visit(MaterializeExpression expression) { + if (expression instanceof MaterializeConstant) { + visit((MaterializeConstant) expression); + } else if (expression instanceof MaterializePostfixOperation) { + visit((MaterializePostfixOperation) expression); + } else if (expression instanceof MaterializeColumnValue) { + visit((MaterializeColumnValue) expression); + } else if (expression instanceof MaterializePrefixOperation) { + visit((MaterializePrefixOperation) expression); + } else if (expression instanceof MaterializeSelect) { + visit((MaterializeSelect) expression); + } else if (expression instanceof MaterializeOrderByTerm) { + visit((MaterializeOrderByTerm) expression); + } else if (expression instanceof MaterializeFunction) { + visit((MaterializeFunction) expression); + } else if (expression instanceof MaterializeCastOperation) { + visit((MaterializeCastOperation) expression); + } else if (expression instanceof MaterializeBetweenOperation) { + visit((MaterializeBetweenOperation) expression); + } else if (expression instanceof MaterializeInOperation) { + visit((MaterializeInOperation) expression); + } else if (expression instanceof MaterializeAggregate) { + visit((MaterializeAggregate) expression); + } else if (expression instanceof MaterializePostfixText) { + visit((MaterializePostfixText) expression); + } else if (expression instanceof MaterializeSimilarTo) { + visit((MaterializeSimilarTo) expression); + } else if (expression instanceof MaterializePOSIXRegularExpression) { + visit((MaterializePOSIXRegularExpression) expression); + } else if (expression instanceof MaterializeFromTable) { + visit((MaterializeFromTable) expression); + } else if (expression instanceof MaterializeSubquery) { + visit((MaterializeSubquery) expression); + } else if (expression instanceof MaterializeLikeOperation) { + visit((MaterializeLikeOperation) expression); + } else { + throw new AssertionError(expression); + } + } + + static String asString(MaterializeExpression expr) { + MaterializeToStringVisitor visitor = new MaterializeToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + static String asExpectedValues(MaterializeExpression expr) { + MaterializeExpectedValueVisitor v = new MaterializeExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + + static String getExpressionAsString(MaterializeGlobalState globalState, MaterializeDataType type, + List columns) { + MaterializeExpression expression = MaterializeExpressionGenerator.generateExpression(globalState, columns, + type); + MaterializeToStringVisitor visitor = new MaterializeToStringVisitor(); + visitor.visit(expression); + return visitor.get(); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeAggregate.java b/src/sqlancer/materialize/ast/MaterializeAggregate.java new file mode 100644 index 000000000..e9dc638f3 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeAggregate.java @@ -0,0 +1,58 @@ +package sqlancer.materialize.ast; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; + +/** + * @see Built-in Aggregate Functions + */ +public class MaterializeAggregate extends FunctionNode + implements MaterializeExpression { + + public enum MaterializeAggregateFunction { + AVG(MaterializeDataType.INT, MaterializeDataType.FLOAT, MaterializeDataType.REAL, MaterializeDataType.DECIMAL), + BIT_AND(MaterializeDataType.INT), BIT_OR(MaterializeDataType.INT), BOOL_AND(MaterializeDataType.BOOLEAN), + BOOL_OR(MaterializeDataType.BOOLEAN), COUNT(MaterializeDataType.INT), MAX, MIN, + SUM(MaterializeDataType.INT, MaterializeDataType.FLOAT, MaterializeDataType.REAL, MaterializeDataType.DECIMAL); + + private MaterializeDataType[] supportedReturnTypes; + + MaterializeAggregateFunction(MaterializeDataType... supportedReturnTypes) { + this.supportedReturnTypes = supportedReturnTypes.clone(); + } + + public List getTypes(MaterializeDataType returnType) { + return Arrays.asList(returnType); + } + + public boolean supportsReturnType(MaterializeDataType returnType) { + return Arrays.asList(supportedReturnTypes).stream().anyMatch(t -> t == returnType) + || supportedReturnTypes.length == 0; + } + + public static List getAggregates(MaterializeDataType type) { + return Arrays.asList(values()).stream().filter(p -> p.supportsReturnType(type)) + .collect(Collectors.toList()); + } + + public MaterializeDataType getRandomReturnType() { + if (supportedReturnTypes.length == 0) { + return Randomly.fromOptions(MaterializeDataType.getRandomType()); + } else { + return Randomly.fromOptions(supportedReturnTypes); + } + } + + } + + public MaterializeAggregate(List args, MaterializeAggregateFunction func) { + super(func, args); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeAlias.java b/src/sqlancer/materialize/ast/MaterializeAlias.java new file mode 100644 index 000000000..ec5445125 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeAlias.java @@ -0,0 +1,35 @@ +package sqlancer.materialize.ast; + +import sqlancer.common.visitor.UnaryOperation; + +public class MaterializeAlias implements UnaryOperation, MaterializeExpression { + + private final MaterializeExpression expr; + private final String alias; + + public MaterializeAlias(MaterializeExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + @Override + public MaterializeExpression getExpression() { + return expr; + } + + @Override + public String getOperatorRepresentation() { + return " as " + alias; + } + + @Override + public OperatorKind getOperatorKind() { + return OperatorKind.POSTFIX; + } + + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBetweenOperation.java b/src/sqlancer/materialize/ast/MaterializeBetweenOperation.java new file mode 100644 index 000000000..dcb7b7cdb --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBetweenOperation.java @@ -0,0 +1,66 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryComparisonOperation.MaterializeBinaryComparisonOperator; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation.BinaryLogicalOperator; + +public final class MaterializeBetweenOperation implements MaterializeExpression { + + private final MaterializeExpression expr; + private final MaterializeExpression left; + private final MaterializeExpression right; + private final boolean isSymmetric; + + public MaterializeBetweenOperation(MaterializeExpression expr, MaterializeExpression left, + MaterializeExpression right, boolean symmetric) { + this.expr = expr; + this.left = left; + this.right = right; + isSymmetric = symmetric; + } + + public MaterializeExpression getExpr() { + return expr; + } + + public MaterializeExpression getLeft() { + return left; + } + + public MaterializeExpression getRight() { + return right; + } + + public boolean isSymmetric() { + return isSymmetric; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeBinaryComparisonOperation leftComparison = new MaterializeBinaryComparisonOperation(left, expr, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryComparisonOperation rightComparison = new MaterializeBinaryComparisonOperation(expr, right, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryLogicalOperation andOperation = new MaterializeBinaryLogicalOperation(leftComparison, + rightComparison, MaterializeBinaryLogicalOperation.BinaryLogicalOperator.AND); + if (isSymmetric) { + MaterializeBinaryComparisonOperation leftComparison2 = new MaterializeBinaryComparisonOperation(right, expr, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryComparisonOperation rightComparison2 = new MaterializeBinaryComparisonOperation(expr, left, + MaterializeBinaryComparisonOperator.LESS_EQUALS); + MaterializeBinaryLogicalOperation andOperation2 = new MaterializeBinaryLogicalOperation(leftComparison2, + rightComparison2, MaterializeBinaryLogicalOperation.BinaryLogicalOperator.AND); + MaterializeBinaryLogicalOperation orOp = new MaterializeBinaryLogicalOperation(andOperation, andOperation2, + BinaryLogicalOperator.OR); + return orOp.getExpectedValue(); + } else { + return andOperation.getExpectedValue(); + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java new file mode 100644 index 000000000..0ba48924f --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryArithmeticOperation.java @@ -0,0 +1,103 @@ +package sqlancer.materialize.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryArithmeticOperation.MaterializeBinaryOperator; + +public class MaterializeBinaryArithmeticOperation + extends BinaryOperatorNode implements MaterializeExpression { + + public enum MaterializeBinaryOperator implements Operator { + + ADDITION("+") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> l + r); + } + + }, + SUBTRACTION("-") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + + } + + }, + MODULO("%") { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + return applyBitOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + + } + }; + + private String textRepresentation; + + private static MaterializeConstant applyBitOperation(MaterializeConstant left, MaterializeConstant right, + BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + long leftVal = left.cast(MaterializeDataType.INT).asInt(); + long rightVal = right.cast(MaterializeDataType.INT).asInt(); + long value = op.apply(leftVal, rightVal); + return MaterializeConstant.createIntConstant(value); + } + } + + MaterializeBinaryOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public abstract MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right); + + public static MaterializeBinaryOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public MaterializeBinaryArithmeticOperation(MaterializeExpression left, MaterializeExpression right, + MaterializeBinaryOperator op) { + super(left, right, op); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpected = getLeft().getExpectedValue(); + MaterializeConstant rightExpected = getRight().getExpectedValue(); + if (leftExpected == null || rightExpected == null) { + return null; + } + return getOp().apply(leftExpected, rightExpected); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java new file mode 100644 index 000000000..8f143da76 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryBitOperation.java @@ -0,0 +1,46 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryBitOperation.MaterializeBinaryBitOperator; + +public class MaterializeBinaryBitOperation extends + BinaryOperatorNode implements MaterializeExpression { + + public enum MaterializeBinaryBitOperator implements Operator { + BITWISE_AND("&"), // + BITWISE_OR("|"), // + BITWISE_XOR("#"), // + BITWISE_SHIFT_LEFT("<<"), // + BITWISE_SHIFT_RIGHT(">>"); + + private String text; + + MaterializeBinaryBitOperator(String text) { + this.text = text; + } + + public static MaterializeBinaryBitOperator getRandom() { + return Randomly.fromOptions(MaterializeBinaryBitOperator.values()); + } + + @Override + public String getTextRepresentation() { + return text; + } + + } + + public MaterializeBinaryBitOperation(MaterializeBinaryBitOperator op, MaterializeExpression left, + MaterializeExpression right) { + super(left, right, op); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java new file mode 100644 index 000000000..25121935a --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryComparisonOperation.java @@ -0,0 +1,121 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryComparisonOperation.MaterializeBinaryComparisonOperator; + +public class MaterializeBinaryComparisonOperation + extends BinaryOperatorNode + implements MaterializeExpression { + + public enum MaterializeBinaryComparisonOperator implements Operator { + EQUALS("=") { + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + return leftVal.isEquals(rightVal); + } + }, + NOT_EQUALS("!=") { + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant isEquals = leftVal.isEquals(rightVal); + if (isEquals.isBoolean()) { + return MaterializeConstant.createBooleanConstant(!isEquals.asBoolean()); + } + return isEquals; + } + }, + LESS("<") { + + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + return leftVal.isLessThan(rightVal); + } + }, + LESS_EQUALS("<=") { + + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant lessThan = leftVal.isLessThan(rightVal); + if (lessThan.isBoolean() && !lessThan.asBoolean()) { + return leftVal.isEquals(rightVal); + } else { + return lessThan; + } + } + }, + GREATER(">") { + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return MaterializeConstant.createFalse(); + } else { + MaterializeConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return MaterializeConstant.createNullConstant(); + } + return MaterializePrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + }, + GREATER_EQUALS(">=") { + + @Override + public MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal) { + MaterializeConstant equals = leftVal.isEquals(rightVal); + if (equals.isBoolean() && equals.asBoolean()) { + return MaterializeConstant.createTrue(); + } else { + MaterializeConstant applyLess = leftVal.isLessThan(rightVal); + if (applyLess.isNull()) { + return MaterializeConstant.createNullConstant(); + } + return MaterializePrefixOperation.PrefixOperator.NOT.getExpectedValue(applyLess); + } + } + + }; + + private final String textRepresentation; + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + MaterializeBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public abstract MaterializeConstant getExpectedValue(MaterializeConstant leftVal, MaterializeConstant rightVal); + + public static MaterializeBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(MaterializeBinaryComparisonOperator.values()); + } + + } + + public MaterializeBinaryComparisonOperation(MaterializeExpression left, MaterializeExpression right, + MaterializeBinaryComparisonOperator op) { + super(left, right, op); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpectedValue = getLeft().getExpectedValue(); + MaterializeConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java new file mode 100644 index 000000000..8b8e457e2 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryLogicalOperation.java @@ -0,0 +1,89 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation.BinaryLogicalOperator; + +public class MaterializeBinaryLogicalOperation extends BinaryOperatorNode + implements MaterializeExpression { + + public enum BinaryLogicalOperator implements Operator { + AND { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + MaterializeConstant leftBool = left.cast(MaterializeDataType.BOOLEAN); + MaterializeConstant rightBool = right.cast(MaterializeDataType.BOOLEAN); + if (leftBool.isNull()) { + if (rightBool.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + if (rightBool.asBoolean()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant.createFalse(); + } + } + } else if (!leftBool.asBoolean()) { + return MaterializeConstant.createFalse(); + } + assert leftBool.asBoolean(); + if (rightBool.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant.createBooleanConstant(rightBool.isBoolean() && rightBool.asBoolean()); + } + } + }, + OR { + @Override + public MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right) { + MaterializeConstant leftBool = left.cast(MaterializeDataType.BOOLEAN); + MaterializeConstant rightBool = right.cast(MaterializeDataType.BOOLEAN); + if (leftBool.isBoolean() && leftBool.asBoolean()) { + return MaterializeConstant.createTrue(); + } + if (rightBool.isBoolean() && rightBool.asBoolean()) { + return MaterializeConstant.createTrue(); + } + if (leftBool.isNull() || rightBool.isNull()) { + return MaterializeConstant.createNullConstant(); + } + return MaterializeConstant.createFalse(); + } + }; + + public abstract MaterializeConstant apply(MaterializeConstant left, MaterializeConstant right); + + public static BinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + + public MaterializeBinaryLogicalOperation(MaterializeExpression left, MaterializeExpression right, + BinaryLogicalOperator op) { + super(left, right, op); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpectedValue = getLeft().getExpectedValue(); + MaterializeConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java b/src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java new file mode 100644 index 000000000..cf1ef66ef --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeBinaryRangeOperation.java @@ -0,0 +1,74 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryNode; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeBinaryRangeOperation extends BinaryNode + implements MaterializeExpression { + + private final String op; + + public enum MaterializeBinaryRangeOperator implements Operator { + UNION("+"), INTERSECTION("*"), DIFFERENCE("-"); + + private final String textRepresentation; + + MaterializeBinaryRangeOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + public static MaterializeBinaryRangeOperator getRandom() { + return Randomly.fromOptions(values()); + } + + } + + public enum MaterializeBinaryRangeComparisonOperator { + CONTAINS_RANGE_OR_ELEMENT("@>"), RANGE_OR_ELEMENT_IS_CONTAINED("<@"), OVERLAP("&&"), STRICT_LEFT_OF("<<"), + STRICT_RIGHT_OF(">>"); + + private final String textRepresentation; + + MaterializeBinaryRangeComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public String getTextRepresentation() { + return textRepresentation; + } + + public static MaterializeBinaryRangeComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public MaterializeBinaryRangeOperation(MaterializeBinaryRangeComparisonOperator op, MaterializeExpression left, + MaterializeExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + public MaterializeBinaryRangeOperation(MaterializeBinaryRangeOperator op, MaterializeExpression left, + MaterializeExpression right) { + super(left, right); + this.op = op.getTextRepresentation(); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public String getOperatorRepresentation() { + return op; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeCastOperation.java b/src/sqlancer/materialize/ast/MaterializeCastOperation.java new file mode 100644 index 000000000..ff7e20180 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeCastOperation.java @@ -0,0 +1,45 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeCastOperation implements MaterializeExpression { + + private final MaterializeExpression expression; + private final MaterializeCompoundDataType type; + + public MaterializeCastOperation(MaterializeExpression expression, MaterializeCompoundDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + @Override + public MaterializeDataType getExpressionType() { + return type.getDataType(); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant expectedValue = expression.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type.getDataType()); + } + + public MaterializeExpression getExpression() { + return expression; + } + + public MaterializeDataType getType() { + return type.getDataType(); + } + + public MaterializeCompoundDataType getCompoundType() { + return type; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeColumnValue.java b/src/sqlancer/materialize/ast/MaterializeColumnValue.java new file mode 100644 index 000000000..054f34ffd --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeColumnValue.java @@ -0,0 +1,34 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeColumnValue implements MaterializeExpression { + + private final MaterializeColumn c; + private final MaterializeConstant expectedValue; + + public MaterializeColumnValue(MaterializeColumn c, MaterializeConstant expectedValue) { + this.c = c; + this.expectedValue = expectedValue; + } + + @Override + public MaterializeDataType getExpressionType() { + return c.getType(); + } + + @Override + public MaterializeConstant getExpectedValue() { + return expectedValue; + } + + public static MaterializeColumnValue create(MaterializeColumn c, MaterializeConstant expected) { + return new MaterializeColumnValue(c, expected); + } + + public MaterializeColumn getColumn() { + return c; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeConcatOperation.java b/src/sqlancer/materialize/ast/MaterializeConcatOperation.java new file mode 100644 index 000000000..c1963c871 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeConcatOperation.java @@ -0,0 +1,37 @@ +package sqlancer.materialize.ast; + +import sqlancer.common.ast.BinaryNode; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeConcatOperation extends BinaryNode implements MaterializeExpression { + + public MaterializeConcatOperation(MaterializeExpression left, MaterializeExpression right) { + super(left, right); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.TEXT; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftExpectedValue = getLeft().getExpectedValue(); + MaterializeConstant rightExpectedValue = getRight().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + if (leftExpectedValue.isNull() || rightExpectedValue.isNull()) { + return MaterializeConstant.createNullConstant(); + } + String leftStr = leftExpectedValue.cast(MaterializeDataType.TEXT).getUnquotedTextRepresentation(); + String rightStr = rightExpectedValue.cast(MaterializeDataType.TEXT).getUnquotedTextRepresentation(); + return MaterializeConstant.createTextConstant(leftStr + rightStr); + } + + @Override + public String getOperatorRepresentation() { + return "||"; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeConstant.java b/src/sqlancer/materialize/ast/MaterializeConstant.java new file mode 100644 index 000000000..dbf668806 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeConstant.java @@ -0,0 +1,516 @@ +package sqlancer.materialize.ast; + +import java.math.BigDecimal; + +import sqlancer.IgnoreMeException; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public abstract class MaterializeConstant implements MaterializeExpression { + + public abstract String getTextRepresentation(); + + public abstract String getUnquotedTextRepresentation(); + + public static class BooleanConstant extends MaterializeConstant { + + private final boolean value; + + public BooleanConstant(boolean value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return value ? "TRUE" : "FALSE"; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return MaterializeConstant.createBooleanConstant(value == rightVal.asBoolean()); + } else if (rightVal.isString()) { + return MaterializeConstant + .createBooleanConstant(value == rightVal.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isString()) { + return isLessThan(rightVal.cast(MaterializeDataType.BOOLEAN)); + } else { + assert rightVal.isBoolean(); + return MaterializeConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); + } + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + switch (type) { + case BOOLEAN: + return this; + case INT: + return MaterializeConstant.createIntConstant(value ? 1 : 0); + case TEXT: + return MaterializeConstant.createTextConstant(value ? "true" : "false"); + default: + return null; + } + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + } + + public static class MaterializeNullConstant extends MaterializeConstant { + + @Override + public String getTextRepresentation() { + return "NULL"; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + return MaterializeConstant.createNullConstant(); + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + return MaterializeConstant.createNullConstant(); + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + return MaterializeConstant.createNullConstant(); + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + } + + public static class StringConstant extends MaterializeConstant { + + private final String value; + + public StringConstant(String value) { + this.value = value; + } + + @Override + public String getTextRepresentation() { + return String.format("'%s'", value.replace("'", "''")); + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(MaterializeDataType.INT).isEquals(rightVal.cast(MaterializeDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(MaterializeDataType.BOOLEAN).isEquals(rightVal.cast(MaterializeDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return cast(MaterializeDataType.INT).isLessThan(rightVal.cast(MaterializeDataType.INT)); + } else if (rightVal.isBoolean()) { + return cast(MaterializeDataType.BOOLEAN).isLessThan(rightVal.cast(MaterializeDataType.BOOLEAN)); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + if (type == MaterializeDataType.TEXT) { + return this; + } + String s = value.trim(); + switch (type) { + case BOOLEAN: + try { + return MaterializeConstant.createBooleanConstant(Long.parseLong(s) != 0); + } catch (NumberFormatException e) { + } + switch (s.toUpperCase()) { + case "T": + case "TR": + case "TRU": + case "TRUE": + case "1": + case "YES": + case "YE": + case "Y": + case "ON": + return MaterializeConstant.createTrue(); + case "F": + case "FA": + case "FAL": + case "FALS": + case "FALSE": + case "N": + case "NO": + case "OF": + case "OFF": + default: + return MaterializeConstant.createFalse(); + } + case INT: + try { + return MaterializeConstant.createIntConstant(Long.parseLong(s)); + } catch (NumberFormatException e) { + return MaterializeConstant.createIntConstant(-1); + } + case TEXT: + return this; + default: + return null; + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.TEXT; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public String asString() { + return value; + } + + @Override + public String getUnquotedTextRepresentation() { + return value; + } + + } + + public static class IntConstant extends MaterializeConstant { + + private final long val; + + public IntConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + + @Override + public long asInt() { + return val; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isBoolean()) { + return cast(MaterializeDataType.BOOLEAN).isEquals(rightVal); + } else if (rightVal.isInt()) { + return MaterializeConstant.createBooleanConstant(val == rightVal.asInt()); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(val == rightVal.cast(MaterializeDataType.INT).asInt()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + if (rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return MaterializeConstant.createBooleanConstant(val < rightVal.asInt()); + } else if (rightVal.isBoolean()) { + throw new AssertionError(rightVal); + } else if (rightVal.isString()) { + return MaterializeConstant.createBooleanConstant(val < rightVal.cast(MaterializeDataType.INT).asInt()); + } else { + throw new IgnoreMeException(); + } + + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + switch (type) { + case BOOLEAN: + return MaterializeConstant.createBooleanConstant(val != 0); + case INT: + return this; + case TEXT: + return MaterializeConstant.createTextConstant(String.valueOf(val)); + default: + return null; + } + } + + @Override + public String getUnquotedTextRepresentation() { + return getTextRepresentation(); + } + + } + + public static MaterializeConstant createNullConstant() { + return new MaterializeNullConstant(); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isString() { + return false; + } + + public static MaterializeConstant createIntConstant(long val) { + return new IntConstant(val); + } + + public static MaterializeConstant createBooleanConstant(boolean val) { + return new BooleanConstant(val); + } + + @Override + public MaterializeConstant getExpectedValue() { + return this; + } + + public boolean isNull() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public static MaterializeConstant createFalse() { + return createBooleanConstant(false); + } + + public static MaterializeConstant createTrue() { + return createBooleanConstant(true); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public boolean isBoolean() { + return false; + } + + public abstract MaterializeConstant isEquals(MaterializeConstant rightVal); + + public boolean isInt() { + return false; + } + + protected abstract MaterializeConstant isLessThan(MaterializeConstant rightVal); + + @Override + public String toString() { + return getTextRepresentation(); + } + + public abstract MaterializeConstant cast(MaterializeDataType type); + + public static MaterializeConstant createTextConstant(String string) { + return new StringConstant(string); + } + + public abstract static class MaterializeConstantBase extends MaterializeConstant { + + @Override + public String getUnquotedTextRepresentation() { + return null; + } + + @Override + public MaterializeConstant isEquals(MaterializeConstant rightVal) { + return null; + } + + @Override + protected MaterializeConstant isLessThan(MaterializeConstant rightVal) { + return null; + } + + @Override + public MaterializeConstant cast(MaterializeDataType type) { + return null; + } + } + + public static class DecimalConstant extends MaterializeConstantBase { + + private final BigDecimal val; + + public DecimalConstant(BigDecimal val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.valueOf(val); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.DECIMAL; + } + + } + + public static class FloatConstant extends MaterializeConstantBase { + + private final float val; + + public FloatConstant(float val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.FLOAT; + } + + } + + public static class DoubleConstant extends MaterializeConstantBase { + + private final double val; + + public DoubleConstant(double val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + if (Double.isFinite(val)) { + return String.valueOf(val); + } else { + return "'" + val + "'"; + } + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.FLOAT; + } + + } + + public static class BitConstant extends MaterializeConstantBase { + + private final long val; + + public BitConstant(long val) { + this.val = val; + } + + @Override + public String getTextRepresentation() { + return String.format("%d", val); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BIT; + } + + } + + public static MaterializeConstant createDecimalConstant(BigDecimal bigDecimal) { + return new DecimalConstant(bigDecimal); + } + + public static MaterializeConstant createFloatConstant(float val) { + return new FloatConstant(val); + } + + public static MaterializeConstant createDoubleConstant(double val) { + return new DoubleConstant(val); + } + + public static MaterializeExpression createBitConstant(long integer) { + return new BitConstant(integer); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeExpression.java b/src/sqlancer/materialize/ast/MaterializeExpression.java new file mode 100644 index 000000000..8a42ad3f6 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeExpression.java @@ -0,0 +1,14 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public interface MaterializeExpression { + + default MaterializeDataType getExpressionType() { + return null; + } + + default MaterializeConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializeFunction.java b/src/sqlancer/materialize/ast/MaterializeFunction.java new file mode 100644 index 000000000..24fdce866 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeFunction.java @@ -0,0 +1,256 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeFunction implements MaterializeExpression { + + private final String func; + private final MaterializeExpression[] args; + private final MaterializeDataType returnType; + private MaterializeFunctionWithResult functionWithKnownResult; + + public MaterializeFunction(MaterializeFunctionWithResult func, MaterializeDataType returnType, + MaterializeExpression... args) { + functionWithKnownResult = func; + this.func = func.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public MaterializeFunction(MaterializeFunctionWithUnknownResult f, MaterializeDataType returnType, + MaterializeExpression... args) { + this.func = f.getName(); + this.returnType = returnType; + this.args = args.clone(); + } + + public String getFunctionName() { + return func; + } + + public MaterializeExpression[] getArguments() { + return args.clone(); + } + + public enum MaterializeFunctionWithResult { + ABS(1, "abs") { + + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant + .createIntConstant(Math.abs(evaluatedArgs[0].cast(MaterializeDataType.INT).asInt())); + } + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { returnType }; + } + + }, + LOWER(1, "lower") { + + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return MaterializeConstant.createTextConstant(text.toLowerCase()); + } + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.TEXT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { MaterializeDataType.TEXT }; + } + + }, + LENGTH(1, "length") { + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } + String text = evaluatedArgs[0].asString(); + return MaterializeConstant.createIntConstant(text.length()); + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { MaterializeDataType.TEXT }; + } + }, + UPPER(1, "upper") { + + @Override + public MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args) { + if (evaluatedArgs[0].isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + String text = evaluatedArgs[0].asString(); + return MaterializeConstant.createTextConstant(text.toUpperCase()); + } + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.TEXT; + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return new MaterializeDataType[] { MaterializeDataType.TEXT }; + } + + }, + NUM_NONNULLS(1, "num_nonnulls") { + @Override + public MaterializeConstant apply(MaterializeConstant[] args, MaterializeExpression... origArgs) { + int nr = 0; + for (MaterializeConstant c : args) { + if (!c.isNull()) { + nr++; + } + } + return MaterializeConstant.createIntConstant(nr); + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }, + NUM_NULLS(1, "num_nulls") { + @Override + public MaterializeConstant apply(MaterializeConstant[] args, MaterializeExpression... origArgs) { + int nr = 0; + for (MaterializeConstant c : args) { + if (c.isNull()) { + nr++; + } + } + return MaterializeConstant.createIntConstant(nr); + } + + @Override + public MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, int nrArguments) { + return getRandomTypes(nrArguments); + } + + @Override + public boolean supportsReturnType(MaterializeDataType type) { + return type == MaterializeDataType.INT; + } + + @Override + public boolean isVariadic() { + return true; + } + + }; + + private String functionName; + final int nrArgs; + private final boolean variadic; + + public MaterializeDataType[] getRandomTypes(int nr) { + MaterializeDataType[] types = new MaterializeDataType[nr]; + for (int i = 0; i < types.length; i++) { + types[i] = MaterializeDataType.getRandomType(); + } + return types; + } + + MaterializeFunctionWithResult(int nrArgs, String functionName) { + this.nrArgs = nrArgs; + this.functionName = functionName; + this.variadic = false; + } + + /** + * Gets the number of arguments if the function is non-variadic. If the function is variadic, the minimum number + * of arguments is returned. + * + * @return the number of arguments + */ + public int getNrArgs() { + return nrArgs; + } + + public abstract MaterializeConstant apply(MaterializeConstant[] evaluatedArgs, MaterializeExpression... args); + + @Override + public String toString() { + return functionName; + } + + public boolean isVariadic() { + return variadic; + } + + public String getName() { + return functionName; + } + + public abstract boolean supportsReturnType(MaterializeDataType type); + + public abstract MaterializeDataType[] getInputTypesForReturnType(MaterializeDataType returnType, + int nrArguments); + + public boolean checkArguments(MaterializeExpression... constants) { + return true; + } + + } + + @Override + public MaterializeConstant getExpectedValue() { + if (functionWithKnownResult == null) { + return null; + } + MaterializeConstant[] constants = new MaterializeConstant[args.length]; + for (int i = 0; i < constants.length; i++) { + constants[i] = args[i].getExpectedValue(); + if (constants[i] == null) { + return null; + } + } + return functionWithKnownResult.apply(constants, args); + } + + @Override + public MaterializeDataType getExpressionType() { + return returnType; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java b/src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java new file mode 100644 index 000000000..fb34c2247 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeFunctionWithUnknownResult.java @@ -0,0 +1,148 @@ +package sqlancer.materialize.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public enum MaterializeFunctionWithUnknownResult { + + CURRENT_DATABASE("current_database", MaterializeDataType.TEXT), // name + CURRENT_SCHEMA("current_schema", MaterializeDataType.TEXT), // name + PG_BACKEND_PID("pg_backend_pid", MaterializeDataType.INT), + PG_CURRENT_LOGFILE("pg_current_logfile", MaterializeDataType.TEXT), + PG_IS_OTHER_TEMP_SCHEMA("pg_is_other_temp_schema", MaterializeDataType.BOOLEAN), + PG_JIT_AVAILABLE("pg_jit_available", MaterializeDataType.BOOLEAN), + PG_NOTIFICATION_QUEUE_USAGE("pg_notification_queue_usage", MaterializeDataType.REAL), + PG_TRIGGER_DEPTH("pg_trigger_depth", MaterializeDataType.INT), VERSION("version", MaterializeDataType.TEXT), + + // + TO_CHAR("to_char", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT) { + @Override + public MaterializeExpression[] getArguments(MaterializeDataType returnType, MaterializeExpressionGenerator gen, + int depth) { + MaterializeExpression[] args = super.getArguments(returnType, gen, depth); + args[0] = gen.generateExpression(MaterializeDataType.getRandomType()); + return args; + } + }, + + // String functions + ASCII("ascii", MaterializeDataType.INT, MaterializeDataType.TEXT), + BTRIM("btrim", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + CHR("chr", MaterializeDataType.TEXT, MaterializeDataType.INT), + CONVERT_FROM("convert_from", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT) { + @Override + public MaterializeExpression[] getArguments(MaterializeDataType returnType, MaterializeExpressionGenerator gen, + int depth) { + MaterializeExpression[] args = super.getArguments(returnType, gen, depth); + args[1] = MaterializeConstant.createTextConstant(Randomly.fromOptions("UTF8", "LATIN1")); + return args; + } + }, + INITCAP("initcap", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + LEFT("left", MaterializeDataType.TEXT, MaterializeDataType.INT, MaterializeDataType.TEXT), + LOWER("lower", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + MD5("md5", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + UPPER("upper", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + QUOTE_LITERAL("quote_literal", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + QUOTE_IDENT("quote_ident", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + REGEX_REPLACE("regex_replace", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + REPLACE("replace", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + REVERSE("reverse", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + RIGHT("right", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.INT), + RPAD("rpad", MaterializeDataType.TEXT, MaterializeDataType.INT, MaterializeDataType.TEXT), + RTRIM("rtrim", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + SPLIT_PART("split_part", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.INT), + STRPOS("strpos", MaterializeDataType.INT, MaterializeDataType.TEXT, MaterializeDataType.TEXT), + SUBSTR("substr", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.INT, + MaterializeDataType.INT), + TO_ASCII("to_ascii", MaterializeDataType.TEXT, MaterializeDataType.TEXT), + TO_HEX("to_hex", MaterializeDataType.INT, MaterializeDataType.TEXT), + TRANSLATE("translate", MaterializeDataType.TEXT, MaterializeDataType.TEXT, MaterializeDataType.TEXT, + MaterializeDataType.TEXT), + // mathematical functions + ABS("abs", MaterializeDataType.REAL, MaterializeDataType.REAL), + CBRT("cbrt", MaterializeDataType.REAL, MaterializeDataType.REAL), CEILING("ceiling", MaterializeDataType.REAL), // + DEGREES("degrees", MaterializeDataType.REAL), EXP("exp", MaterializeDataType.REAL), + LN("ln", MaterializeDataType.REAL), LOG("log", MaterializeDataType.REAL), + LOG2("log", MaterializeDataType.REAL, MaterializeDataType.REAL), PI("pi", MaterializeDataType.REAL), + POWER("power", MaterializeDataType.REAL, MaterializeDataType.REAL), + TRUNC("trunc", MaterializeDataType.REAL, MaterializeDataType.INT), + TRUNC2("trunc", MaterializeDataType.REAL, MaterializeDataType.INT, MaterializeDataType.REAL), + FLOOR("floor", MaterializeDataType.REAL), + + // trigonometric functions - complete + ACOS("acos", MaterializeDataType.REAL), // + ACOSD("acosd", MaterializeDataType.REAL), // + ASIN("asin", MaterializeDataType.REAL), // + ASIND("asind", MaterializeDataType.REAL), // + ATAN("atan", MaterializeDataType.REAL), // + ATAND("atand", MaterializeDataType.REAL), // + ATAN2("atan2", MaterializeDataType.REAL, MaterializeDataType.REAL), // + ATAN2D("atan2d", MaterializeDataType.REAL, MaterializeDataType.REAL), // + COS("cos", MaterializeDataType.REAL), // + COSD("cosd", MaterializeDataType.REAL), // + COT("cot", MaterializeDataType.REAL), // + COTD("cotd", MaterializeDataType.REAL), // + SIN("sin", MaterializeDataType.REAL), // + SIND("sind", MaterializeDataType.REAL), // + TAN("tan", MaterializeDataType.REAL), // + TAND("tand", MaterializeDataType.REAL), // + + // hyperbolic functions - complete + SINH("sinh", MaterializeDataType.REAL), // + COSH("cosh", MaterializeDataType.REAL), // + TANH("tanh", MaterializeDataType.REAL), // + ASINH("asinh", MaterializeDataType.REAL), // + ACOSH("acosh", MaterializeDataType.REAL), // + ATANH("atanh", MaterializeDataType.REAL), // + + GET_BIT("get_bit", MaterializeDataType.INT, MaterializeDataType.TEXT, MaterializeDataType.INT), + GET_BYTE("get_byte", MaterializeDataType.INT, MaterializeDataType.TEXT, MaterializeDataType.INT), + + GET_COLUMN_SIZE("get_column_size", MaterializeDataType.INT, MaterializeDataType.TEXT); + + private String functionName; + private MaterializeDataType returnType; + private MaterializeDataType[] argTypes; + + MaterializeFunctionWithUnknownResult(String functionName, MaterializeDataType returnType, + MaterializeDataType... indexType) { + this.functionName = functionName; + this.returnType = returnType; + this.argTypes = indexType.clone(); + + } + + public boolean isCompatibleWithReturnType(MaterializeDataType t) { + return t == returnType; + } + + public MaterializeExpression[] getArguments(MaterializeDataType returnType, MaterializeExpressionGenerator gen, + int depth) { + MaterializeExpression[] args = new MaterializeExpression[argTypes.length]; + for (int i = 0; i < args.length; i++) { + args[i] = gen.generateExpression(depth, argTypes[i]); + } + return args; + + } + + public String getName() { + return functionName; + } + + public static List getSupportedFunctions(MaterializeDataType type) { + List functions = new ArrayList<>(); + for (MaterializeFunctionWithUnknownResult func : values()) { + if (func.isCompatibleWithReturnType(type)) { + functions.add(func); + } + } + return functions; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeInOperation.java b/src/sqlancer/materialize/ast/MaterializeInOperation.java new file mode 100644 index 000000000..270a973dc --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeInOperation.java @@ -0,0 +1,66 @@ +package sqlancer.materialize.ast; + +import java.util.List; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeInOperation implements MaterializeExpression { + + private final MaterializeExpression expr; + private final List listElements; + private final boolean isTrue; + + public MaterializeInOperation(MaterializeExpression expr, List listElements, + boolean isTrue) { + this.expr = expr; + this.listElements = listElements; + this.isTrue = isTrue; + } + + public MaterializeExpression getExpr() { + return expr; + } + + public List getListElements() { + return listElements; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftValue = expr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return MaterializeConstant.createNullConstant(); + } + boolean isNull = false; + for (MaterializeExpression expr : getListElements()) { + MaterializeConstant rightExpectedValue = expr.getExpectedValue(); + if (rightExpectedValue == null) { + return null; + } + if (rightExpectedValue.isNull()) { + isNull = true; + } else if (rightExpectedValue.isEquals(this.expr.getExpectedValue()).isBoolean() + && rightExpectedValue.isEquals(this.expr.getExpectedValue()).asBoolean()) { + return MaterializeConstant.createBooleanConstant(isTrue); + } + } + + if (isNull) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant.createBooleanConstant(!isTrue); + } + } + + public boolean isTrue() { + return isTrue; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializeJoin.java b/src/sqlancer/materialize/ast/MaterializeJoin.java new file mode 100644 index 000000000..967cdf420 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeJoin.java @@ -0,0 +1,50 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeJoin implements MaterializeExpression { + + public enum MaterializeJoinType { + INNER, LEFT, RIGHT, FULL, CROSS; + + public static MaterializeJoinType getRandom() { + return Randomly.fromOptions(values()); + } + + } + + private final MaterializeExpression tableReference; + private final MaterializeExpression onClause; + private final MaterializeJoinType type; + + public MaterializeJoin(MaterializeExpression tableReference, MaterializeExpression onClause, + MaterializeJoinType type) { + this.tableReference = tableReference; + this.onClause = onClause; + this.type = type; + } + + public MaterializeExpression getTableReference() { + return tableReference; + } + + public MaterializeExpression getOnClause() { + return onClause; + } + + public MaterializeJoinType getType() { + return type; + } + + @Override + public MaterializeDataType getExpressionType() { + throw new AssertionError(); + } + + @Override + public MaterializeConstant getExpectedValue() { + throw new AssertionError(); + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeLikeOperation.java b/src/sqlancer/materialize/ast/MaterializeLikeOperation.java new file mode 100644 index 000000000..b14209993 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeLikeOperation.java @@ -0,0 +1,38 @@ +package sqlancer.materialize.ast; + +import sqlancer.LikeImplementationHelper; +import sqlancer.common.ast.BinaryNode; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeLikeOperation extends BinaryNode implements MaterializeExpression { + + public MaterializeLikeOperation(MaterializeExpression left, MaterializeExpression right) { + super(left, right); + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant leftVal = getLeft().getExpectedValue(); + MaterializeConstant rightVal = getRight().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } + if (leftVal.isNull() || rightVal.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + boolean val = LikeImplementationHelper.match(leftVal.asString(), rightVal.asString(), 0, 0, true); + return MaterializeConstant.createBooleanConstant(val); + } + } + + @Override + public String getOperatorRepresentation() { + return "LIKE"; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeOrderByTerm.java b/src/sqlancer/materialize/ast/MaterializeOrderByTerm.java new file mode 100644 index 000000000..d4704c87f --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeOrderByTerm.java @@ -0,0 +1,42 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeOrderByTerm implements MaterializeExpression { + + private final MaterializeOrder order; + private final MaterializeExpression expr; + + public enum MaterializeOrder { + ASC, DESC; + + public static MaterializeOrder getRandomOrder() { + return Randomly.fromOptions(MaterializeOrder.values()); + } + } + + public MaterializeOrderByTerm(MaterializeExpression expr, MaterializeOrder order) { + this.expr = expr; + this.order = order; + } + + public MaterializeOrder getOrder() { + return order; + } + + public MaterializeExpression getExpr() { + return expr; + } + + @Override + public MaterializeConstant getExpectedValue() { + throw new AssertionError(this); + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java b/src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java new file mode 100644 index 000000000..127bafe79 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePOSIXRegularExpression.java @@ -0,0 +1,65 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePOSIXRegularExpression implements MaterializeExpression { + + private MaterializeExpression string; + private MaterializeExpression regex; + private POSIXRegex op; + + public enum POSIXRegex implements Operator { + MATCH_CASE_SENSITIVE("~"), MATCH_CASE_INSENSITIVE("~*"), NOT_MATCH_CASE_SENSITIVE("!~"), + NOT_MATCH_CASE_INSENSITIVE("!~*"); + + private String repr; + + POSIXRegex(String repr) { + this.repr = repr; + } + + public String getStringRepresentation() { + return repr; + } + + public static POSIXRegex getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + + public MaterializePOSIXRegularExpression(MaterializeExpression string, MaterializeExpression regex, POSIXRegex op) { + this.string = string; + this.regex = regex; + this.op = op; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + return null; + } + + public MaterializeExpression getRegex() { + return regex; + } + + public MaterializeExpression getString() { + return string; + } + + public POSIXRegex getOp() { + return op; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializePostfixOperation.java b/src/sqlancer/materialize/ast/MaterializePostfixOperation.java new file mode 100644 index 000000000..7bee2e38c --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePostfixOperation.java @@ -0,0 +1,151 @@ +package sqlancer.materialize.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePostfixOperation implements MaterializeExpression { + + private final MaterializeExpression expr; + private final PostfixOperator op; + private final String operatorTextRepresentation; + + public enum PostfixOperator implements Operator { + IS_NULL("IS NULL", "ISNULL") { + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return MaterializeDataType.values(); + } + + }, + IS_UNKNOWN("IS UNKNOWN") { + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + }, + + IS_NOT_NULL("IS NOT NULL") { + + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return MaterializeDataType.values(); + } + + }, + IS_NOT_UNKNOWN("IS NOT UNKNOWN") { + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + return MaterializeConstant.createBooleanConstant(!expectedValue.isNull()); + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + }, + IS_TRUE("IS TRUE") { + + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + return MaterializeConstant.createFalse(); + } else { + return MaterializeConstant + .createBooleanConstant(expectedValue.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + + }, + IS_FALSE("IS FALSE") { + + @Override + public MaterializeConstant apply(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + return MaterializeConstant.createFalse(); + } else { + return MaterializeConstant + .createBooleanConstant(!expectedValue.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } + } + + @Override + public MaterializeDataType[] getInputDataTypes() { + return new MaterializeDataType[] { MaterializeDataType.BOOLEAN }; + } + + }; + + private String[] textRepresentations; + + PostfixOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + public abstract MaterializeConstant apply(MaterializeConstant expectedValue); + + public abstract MaterializeDataType[] getInputDataTypes(); + + public static PostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + } + + public MaterializePostfixOperation(MaterializeExpression expr, PostfixOperator op) { + this.expr = expr; + this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); + this.op = op; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.apply(expectedValue); + } + + public String getOperatorTextRepresentation() { + return operatorTextRepresentation; + } + + public static MaterializeExpression create(MaterializeExpression expr, PostfixOperator op) { + return new MaterializePostfixOperation(expr, op); + } + + public MaterializeExpression getExpression() { + return expr; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializePostfixText.java b/src/sqlancer/materialize/ast/MaterializePostfixText.java new file mode 100644 index 000000000..a64b03c6d --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePostfixText.java @@ -0,0 +1,37 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePostfixText implements MaterializeExpression { + + private final MaterializeExpression expr; + private final String text; + private final MaterializeConstant expectedValue; + private final MaterializeDataType type; + + public MaterializePostfixText(MaterializeExpression expr, String text, MaterializeConstant expectedValue, + MaterializeDataType type) { + this.expr = expr; + this.text = text; + this.expectedValue = expectedValue; + this.type = type; + } + + public MaterializeExpression getExpr() { + return expr; + } + + public String getText() { + return text; + } + + @Override + public MaterializeConstant getExpectedValue() { + return expectedValue; + } + + @Override + public MaterializeDataType getExpressionType() { + return type; + } +} diff --git a/src/sqlancer/materialize/ast/MaterializePrefixOperation.java b/src/sqlancer/materialize/ast/MaterializePrefixOperation.java new file mode 100644 index 000000000..456c65b45 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializePrefixOperation.java @@ -0,0 +1,119 @@ +package sqlancer.materialize.ast; + +import sqlancer.IgnoreMeException; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializePrefixOperation implements MaterializeExpression { + + public enum PrefixOperator implements Operator { + NOT("NOT", MaterializeDataType.BOOLEAN) { + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + protected MaterializeConstant getExpectedValue(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + return MaterializeConstant.createNullConstant(); + } else { + return MaterializeConstant + .createBooleanConstant(!expectedValue.cast(MaterializeDataType.BOOLEAN).asBoolean()); + } + } + }, + UNARY_PLUS("+", MaterializeDataType.INT) { + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + + @Override + protected MaterializeConstant getExpectedValue(MaterializeConstant expectedValue) { + // TODO: actual converts to double precision + return expectedValue; + } + + }, + UNARY_MINUS("-", MaterializeDataType.INT) { + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.INT; + } + + @Override + protected MaterializeConstant getExpectedValue(MaterializeConstant expectedValue) { + if (expectedValue.isNull()) { + // TODO + throw new IgnoreMeException(); + } + if (expectedValue.isInt() && expectedValue.asInt() == Long.MIN_VALUE) { + throw new IgnoreMeException(); + } + try { + return MaterializeConstant.createIntConstant(-expectedValue.asInt()); + } catch (UnsupportedOperationException e) { + return null; + } + } + + }; + + private String textRepresentation; + private MaterializeDataType[] dataTypes; + + PrefixOperator(String textRepresentation, MaterializeDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract MaterializeDataType getExpressionType(); + + protected abstract MaterializeConstant getExpectedValue(MaterializeConstant expectedValue); + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + + private final MaterializeExpression expr; + private final PrefixOperator op; + + public MaterializePrefixOperation(MaterializeExpression expr, PrefixOperator op) { + this.expr = expr; + this.op = op; + } + + @Override + public MaterializeDataType getExpressionType() { + return op.getExpressionType(); + } + + @Override + public MaterializeConstant getExpectedValue() { + MaterializeConstant expectedValue = expr.getExpectedValue(); + if (expectedValue == null) { + return null; + } + return op.getExpectedValue(expectedValue); + } + + public MaterializeDataType[] getInputDataTypes() { + return op.dataTypes; + } + + public String getTextRepresentation() { + return op.textRepresentation; + } + + public MaterializeExpression getExpression() { + return expr; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeSelect.java b/src/sqlancer/materialize/ast/MaterializeSelect.java new file mode 100644 index 000000000..cef761428 --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeSelect.java @@ -0,0 +1,135 @@ +package sqlancer.materialize.ast; + +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; + +public class MaterializeSelect extends SelectBase implements MaterializeExpression { + + private SelectType selectOption = SelectType.ALL; + private List joinClauses = Collections.emptyList(); + private MaterializeExpression distinctOnClause; + private ForClause forClause; + + public enum ForClause { + UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); + + private final String textRepresentation; + + ForClause(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public String getTextRepresentation() { + return textRepresentation; + } + + public static ForClause getRandom() { + return Randomly.fromOptions(values()); + } + } + + public static class MaterializeFromTable implements MaterializeExpression { + private final MaterializeTable t; + private final boolean only; + + public MaterializeFromTable(MaterializeTable t, boolean only) { + this.t = t; + this.only = only; + } + + public MaterializeTable getTable() { + return t; + } + + public boolean isOnly() { + return only; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + } + + public static class MaterializeSubquery implements MaterializeExpression { + private final MaterializeSelect s; + private final String name; + + public MaterializeSubquery(MaterializeSelect s, String name) { + this.s = s; + this.name = name; + } + + public MaterializeSelect getSelect() { + return s; + } + + public String getName() { + return name; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + } + + public enum SelectType { + DISTINCT, ALL; + + public static SelectType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public void setSelectType(SelectType fromOptions) { + this.setSelectOption(fromOptions); + } + + public void setDistinctOnClause(MaterializeExpression distinctOnClause) { + if (selectOption != SelectType.DISTINCT) { + throw new IllegalArgumentException(); + } + this.distinctOnClause = distinctOnClause; + } + + public SelectType getSelectOption() { + return selectOption; + } + + public void setSelectOption(SelectType fromOptions) { + this.selectOption = fromOptions; + } + + @Override + public MaterializeDataType getExpressionType() { + return null; + } + + public void setJoinClauses(List joinStatements) { + this.joinClauses = joinStatements; + + } + + public List getJoinClauses() { + return joinClauses; + } + + public MaterializeExpression getDistinctOnClause() { + return distinctOnClause; + } + + public void setForClause(ForClause forClause) { + this.forClause = forClause; + } + + public ForClause getForClause() { + return forClause; + } + +} diff --git a/src/sqlancer/materialize/ast/MaterializeSimilarTo.java b/src/sqlancer/materialize/ast/MaterializeSimilarTo.java new file mode 100644 index 000000000..dd675050d --- /dev/null +++ b/src/sqlancer/materialize/ast/MaterializeSimilarTo.java @@ -0,0 +1,40 @@ +package sqlancer.materialize.ast; + +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; + +public class MaterializeSimilarTo implements MaterializeExpression { + + private final MaterializeExpression string; + private final MaterializeExpression similarTo; + private final MaterializeExpression escapeCharacter; + + public MaterializeSimilarTo(MaterializeExpression string, MaterializeExpression similarTo, + MaterializeExpression escapeCharacter) { + this.string = string; + this.similarTo = similarTo; + this.escapeCharacter = escapeCharacter; + } + + public MaterializeExpression getString() { + return string; + } + + public MaterializeExpression getSimilarTo() { + return similarTo; + } + + public MaterializeExpression getEscapeCharacter() { + return escapeCharacter; + } + + @Override + public MaterializeDataType getExpressionType() { + return MaterializeDataType.BOOLEAN; + } + + @Override + public MaterializeConstant getExpectedValue() { + return null; + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeCommon.java b/src/sqlancer/materialize/gen/MaterializeCommon.java new file mode 100644 index 000000000..0c4847bed --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeCommon.java @@ -0,0 +1,351 @@ +package sqlancer.materialize.gen; + +import java.util.List; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeProvider; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public final class MaterializeCommon { + + private MaterializeCommon() { + } + + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); + errors.add("but it cannot be referenced from this part of the query"); + errors.add("missing FROM-clause entry for table"); + + errors.add("canceling statement due to statement timeout"); + + errors.add("non-integer constant in GROUP BY"); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("GROUP BY position"); + errors.add("results exceed max size of"); + + errors.add("does not exist"); + errors.add("aggregate functions are not allowed in"); + errors.add("is only defined for finite arguments"); + } + + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.add("is not commutative"); // exclude + errors.add("operator requires run-time type coercion"); // exclude + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.add("You might need to add explicit type casts"); + errors.add("invalid regular expression"); + errors.add("could not determine which collation to use"); + errors.add("invalid regular expression"); + errors.add("operator does not exist"); + errors.add("quantifier operand invalid"); + errors.add("collation mismatch"); + errors.add("collations are not supported"); + errors.add("operator is not unique"); + errors.add("is not a valid binary digit"); + errors.add("invalid hexadecimal digit"); + errors.add("invalid hexadecimal data: odd number of digits"); + errors.add("zero raised to a negative power is undefined"); + errors.add("cannot convert infinity to numeric"); + errors.add("division by zero"); + errors.add("invalid input syntax for type money"); + errors.add("invalid input syntax for type"); + errors.add("cannot cast type"); + errors.add("value overflows numeric format"); + errors.add("numeric field overflow"); + errors.add("LIKE pattern must not end with escape character"); + errors.add("is of type boolean but expression is of type text"); + errors.add("a negative number raised to a non-integer power yields a complex result"); + errors.add("could not determine polymorphic type because input has type unknown"); + errors.add("character number must be positive"); + errors.add("unterminated escape sequence"); + errors.add("cannot be matched"); + errors.add("clause must have type"); // "not" in having doesn't work + errors.add("argument must have type"); // "not" in having doesn't work + errors.add("CAST does not support casting from"); + errors.add("aggregate functions are not allowed in"); + errors.add("only defined for finite arguments"); + errors.add("unable to parse column reference in GROUP BY clause"); // TODO + addToCharFunctionErrors(errors); + addBitStringOperationErrors(errors); + addFunctionErrors(errors); + addCommonRangeExpressionErrors(errors); + addCommonRegexExpressionErrors(errors); + } + + private static void addToCharFunctionErrors(ExpectedErrors errors) { + errors.add("multiple decimal points"); + errors.add("and decimal point together"); + errors.add("multiple decimal points"); + errors.add("cannot use \"S\" twice"); + errors.add("must be ahead of \"PR\""); + errors.add("cannot use \"S\" and \"PL\"/\"MI\"/\"SG\"/\"PR\" together"); + errors.add("cannot use \"S\" and \"SG\" together"); + errors.add("cannot use \"S\" and \"MI\" together"); + errors.add("cannot use \"S\" and \"PL\" together"); + errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); + errors.add("is not a number"); + } + + private static void addBitStringOperationErrors(ExpectedErrors errors) { + errors.add("cannot XOR bit strings of different sizes"); + errors.add("cannot AND bit strings of different sizes"); + errors.add("cannot OR bit strings of different sizes"); + errors.add("must be type boolean, not type text"); + } + + private static void addFunctionErrors(ExpectedErrors errors) { + errors.add("out of valid range"); // get_bit/get_byte + errors.add("cannot take logarithm of a negative number"); + errors.add("cannot take logarithm of zero"); + errors.add("requested character too large for encoding"); // chr + errors.add("null character not permitted"); // chr + errors.add("requested character not valid for encoding"); // chr + errors.add("requested length too large"); // repeat + errors.add("invalid memory alloc request size"); // repeat + errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + } + + private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + errors.add("is not a valid hexadecimal digit"); + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.add("range lower bound must be less than or equal to range upper bound"); + errors.add("result of range difference would not be contiguous"); + errors.add("out of range"); + errors.add("malformed range literal"); + errors.add("result of range union would not be contiguous"); + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.add("value too long for type character"); + errors.add("not found in view targetlist"); + errors.add("CAST does not support casting from"); + } + + public static boolean appendDataType(MaterializeDataType type, StringBuilder sb, boolean allowSerial, + boolean generateOnlyKnown, List opClasses) throws AssertionError { + boolean serial = false; + switch (type) { + case BOOLEAN: + sb.append("boolean"); + break; + case INT: + sb.append(Randomly.fromOptions("smallint", "integer", "bigint")); + break; + case TEXT: + if (Randomly.getBoolean()) { + sb.append("TEXT"); + } else { + if (MaterializeProvider.generateOnlyKnown || Randomly.getBoolean()) { + sb.append("VAR"); + } + sb.append("CHAR"); + sb.append("("); + sb.append(ThreadLocalRandom.current().nextInt(1, 500)); + sb.append(")"); + } + break; + case DECIMAL: + sb.append("DECIMAL"); + break; + case FLOAT: + sb.append("REAL"); + break; + case REAL: + sb.append("FLOAT"); + break; + case BIT: + sb.append("INT"); + break; + default: + throw new AssertionError(type); + } + return serial; + } + + public enum TableConstraints { + CHECK, PRIMARY_KEY, FOREIGN_KEY, EXCLUDE + } + + public static void addTableConstraints(boolean excludePrimaryKey, StringBuilder sb, MaterializeTable table, + MaterializeGlobalState globalState, ExpectedErrors errors) { + // TODO constraint name + List tableConstraints = Randomly.nonEmptySubset(TableConstraints.values()); + if (excludePrimaryKey) { + tableConstraints.remove(TableConstraints.PRIMARY_KEY); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + tableConstraints.remove(TableConstraints.FOREIGN_KEY); + } + for (TableConstraints t : tableConstraints) { + sb.append(", "); + // TODO add index parameters + addTableConstraint(sb, table, globalState, t, errors); + } + } + + public static void addTableConstraint(StringBuilder sb, MaterializeTable table, MaterializeGlobalState globalState, + ExpectedErrors errors) { + addTableConstraint(sb, table, globalState, Randomly.fromOptions(TableConstraints.values()), errors); + } + + private static void addTableConstraint(StringBuilder sb, MaterializeTable table, MaterializeGlobalState globalState, + TableConstraints t, ExpectedErrors errors) { + List randomNonEmptyColumnSubset = table.getRandomNonEmptyColumnSubset(); + List otherColumns; + MaterializeCommon.addCommonExpressionErrors(errors); + switch (t) { + case CHECK: + sb.append("CHECK("); + sb.append(MaterializeVisitor.getExpressionAsString(globalState, MaterializeDataType.BOOLEAN, + table.getColumns())); + sb.append(")"); + errors.add("constraint must be added to child tables too"); + errors.add("missing FROM-clause entry for table"); + break; + case PRIMARY_KEY: + sb.append("PRIMARY KEY("); + sb.append(randomNonEmptyColumnSubset.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + break; + case FOREIGN_KEY: + sb.append("FOREIGN KEY ("); + sb.append(randomNonEmptyColumnSubset.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(") REFERENCES "); + MaterializeTable randomOtherTable = globalState.getSchema().getRandomTable(tab -> !tab.isView()); + sb.append(randomOtherTable.getName()); + if (randomOtherTable.getColumns().size() < randomNonEmptyColumnSubset.size()) { + throw new IgnoreMeException(); + } + otherColumns = randomOtherTable.getRandomNonEmptyColumnSubset(randomNonEmptyColumnSubset.size()); + sb.append("("); + sb.append(otherColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("MATCH FULL", "MATCH SIMPLE")); + } + if (Randomly.getBoolean()) { + sb.append(" ON DELETE "); + errors.add("ERROR: invalid ON DELETE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" ON UPDATE "); + errors.add("invalid ON UPDATE action for foreign key constraint containing generated column"); + deleteOrUpdateAction(sb); + } + if (Randomly.getBoolean()) { + sb.append(" "); + if (Randomly.getBoolean()) { + sb.append("DEFERRABLE"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("INITIALLY DEFERRED", "INITIALLY IMMEDIATE")); + } + } else { + sb.append("NOT DEFERRABLE"); + } + } + break; + case EXCLUDE: + sb.append("EXCLUDE "); + sb.append("("); + // TODO [USING index_method ] + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + appendExcludeElement(sb, globalState, table.getColumns()); + sb.append(" WITH "); + appendOperator(sb, globalState.getOperators()); + } + sb.append(")"); + errors.add("is not valid"); + errors.add("no operator matches"); + errors.add("operator does not exist"); + errors.add("unknown has no default operator class"); + errors.add("exclusion constraints are not supported on partitioned tables"); + errors.add("The exclusion operator must be related to the index operator class for the constraint"); + errors.add("could not create exclusion constraint"); + // TODO: index parameters + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append("("); + sb.append(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, + table.getColumns(), MaterializeDataType.BOOLEAN))); + sb.append(")"); + } + break; + default: + throw new AssertionError(t); + } + } + + private static void appendOperator(StringBuilder sb, List operators) { + sb.append(Randomly.fromList(operators)); + } + + // complete + private static void appendExcludeElement(StringBuilder sb, MaterializeGlobalState globalState, + List columns) { + if (Randomly.getBoolean()) { + // append column name + sb.append(Randomly.fromList(columns).getName()); + } else { + // append expression + sb.append("("); + sb.append(MaterializeVisitor + .asString(MaterializeExpressionGenerator.generateExpression(globalState, columns))); + sb.append(")"); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromList(globalState.getOpClasses())); + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBoolean()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + + private static void deleteOrUpdateAction(StringBuilder sb) { + sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("unable to parse column reference in GROUP BY clause"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in"); + } + + public static String getFreeIndexName(MaterializeSchema s) { + List indexNames = s.getIndexNames(); + String candidateName; + do { + candidateName = DBMSCommon.createIndexName((int) Randomly.getNotCachedInteger(0, 100)); + } while (indexNames.contains(candidateName)); + return candidateName; + } +} diff --git a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java new file mode 100644 index 000000000..ded3e53f4 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java @@ -0,0 +1,38 @@ +package sqlancer.materialize.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public final class MaterializeDeleteGenerator { + + private MaterializeDeleteGenerator() { + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + MaterializeTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("violates foreign key constraint"); + errors.add("violates not-null constraint"); + errors.add("could not determine which collation to use for string comparison"); + StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append(" "); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, + table.getColumns(), MaterializeDataType.BOOLEAN))); + } + MaterializeCommon.addCommonExpressionErrors(errors); + errors.add("out of range"); + errors.add("does not support casting"); + errors.add("invalid input syntax for"); + errors.add("division by zero"); + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java b/src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java new file mode 100644 index 000000000..4acbd329a --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeDropIndexGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.materialize.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeIndex; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; + +public final class MaterializeDropIndexGenerator { + + private MaterializeDropIndexGenerator() { + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + MaterializeTable randomTable = globalState.getSchema().getRandomTable(); + List indexes = randomTable.getIndexes(); + StringBuilder sb = new StringBuilder(); + sb.append("DROP INDEX "); + if (Randomly.getBoolean() || indexes.isEmpty()) { + sb.append("IF EXISTS "); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + if (indexes.isEmpty() || Randomly.getBoolean()) { + sb.append(DBMSCommon.createIndexName(Randomly.smallNumber())); + } else { + sb.append(Randomly.fromList(indexes).getIndexName()); + } + } + } else { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(Randomly.fromList(indexes).getIndexName()); + } + } + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); + } + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot drop desired object(s) because other objects depend on them", + "cannot drop index", "does not exist"), + true); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java new file mode 100644 index 000000000..1396052a2 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -0,0 +1,517 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeProvider; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeRowValue; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; +import sqlancer.materialize.ast.MaterializeBetweenOperation; +import sqlancer.materialize.ast.MaterializeBinaryArithmeticOperation; +import sqlancer.materialize.ast.MaterializeBinaryArithmeticOperation.MaterializeBinaryOperator; +import sqlancer.materialize.ast.MaterializeBinaryBitOperation; +import sqlancer.materialize.ast.MaterializeBinaryBitOperation.MaterializeBinaryBitOperator; +import sqlancer.materialize.ast.MaterializeBinaryComparisonOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation; +import sqlancer.materialize.ast.MaterializeBinaryLogicalOperation.BinaryLogicalOperator; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConcatOperation; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeFunction; +import sqlancer.materialize.ast.MaterializeFunction.MaterializeFunctionWithResult; +import sqlancer.materialize.ast.MaterializeFunctionWithUnknownResult; +import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeLikeOperation; +import sqlancer.materialize.ast.MaterializeOrderByTerm; +import sqlancer.materialize.ast.MaterializeOrderByTerm.MaterializeOrder; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression; +import sqlancer.materialize.ast.MaterializePOSIXRegularExpression.POSIXRegex; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializePrefixOperation.PrefixOperator; + +public class MaterializeExpressionGenerator implements ExpressionGenerator { + + private final int maxDepth; + + private final Randomly r; + + private List columns; + + private MaterializeRowValue rw; + + private boolean expectedResult; + + private MaterializeGlobalState globalState; + + private boolean allowAggregateFunctions; + + private final Map functionsAndTypes; + + private final List allowedFunctionTypes; + + public MaterializeExpressionGenerator(MaterializeGlobalState globalState) { + this.r = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + this.globalState = globalState; + this.functionsAndTypes = globalState.getFunctionsAndTypes(); + this.allowedFunctionTypes = globalState.getAllowedFunctionTypes(); + } + + public MaterializeExpressionGenerator setColumns(List columns) { + this.columns = columns; + return this; + } + + public MaterializeExpressionGenerator setRowValue(MaterializeRowValue rw) { + this.rw = rw; + return this; + } + + public MaterializeExpression generateExpression(int depth) { + return generateExpression(depth, MaterializeDataType.getRandomType()); + } + + public List generateOrderBy() { + List orderBys = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + orderBys.add(new MaterializeOrderByTerm(MaterializeColumnValue.create(Randomly.fromList(columns), null), + MaterializeOrder.getRandomOrder())); + } + return orderBys; + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, LIKE, BETWEEN, IN_OPERATION, + POSIX_REGEX; + } + + private MaterializeExpression generateFunctionWithUnknownResult(int depth, MaterializeDataType type) { + List supportedFunctions = MaterializeFunctionWithUnknownResult + .getSupportedFunctions(type); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + supportedFunctions = supportedFunctions.stream() + .filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (supportedFunctions.isEmpty()) { + throw new IgnoreMeException(); + } + MaterializeFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); + return new MaterializeFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); + } + + private MaterializeExpression generateFunctionWithKnownResult(int depth, MaterializeDataType type) { + List functions = Stream + .of(MaterializeFunction.MaterializeFunctionWithResult.values()).filter(f -> f.supportsReturnType(type)) + .collect(Collectors.toList()); + // filters functions by allowed type (STABLE 's', IMMUTABLE 'i', VOLATILE 'v') + functions = functions.stream().filter(f -> allowedFunctionTypes.contains(functionsAndTypes.get(f.getName()))) + .collect(Collectors.toList()); + if (functions.isEmpty()) { + throw new IgnoreMeException(); + } + MaterializeFunctionWithResult randomFunction = Randomly.fromList(functions); + int nrArgs = randomFunction.getNrArgs(); + if (randomFunction.isVariadic()) { + nrArgs += Randomly.smallNumber(); + } + MaterializeDataType[] argTypes = randomFunction.getInputTypesForReturnType(type, nrArgs); + MaterializeExpression[] args = new MaterializeExpression[nrArgs]; + do { + for (int i = 0; i < args.length; i++) { + args[i] = generateExpression(depth + 1, argTypes[i]); + } + } while (!randomFunction.checkArguments(args)); + return new MaterializeFunction(randomFunction, type, args); + } + + private MaterializeExpression generateBooleanExpression(int depth) { + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (MaterializeProvider.generateOnlyKnown) { + validOptions.remove(BooleanExpression.POSIX_REGEX); + } + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + PostfixOperator random = PostfixOperator.getRandom(); + return MaterializePostfixOperation + .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); + case IN_OPERATION: + return inOperation(depth + 1); + case NOT: + return new MaterializePrefixOperation(generateExpression(depth + 1, MaterializeDataType.BOOLEAN), + PrefixOperator.NOT); + case BINARY_LOGICAL_OPERATOR: + MaterializeExpression first = generateExpression(depth + 1, MaterializeDataType.BOOLEAN); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + first = new MaterializeBinaryLogicalOperation(first, + generateExpression(depth + 1, MaterializeDataType.BOOLEAN), BinaryLogicalOperator.getRandom()); + } + return first; + case BINARY_COMPARISON: + MaterializeDataType dataType = getMeaningfulType(); + return generateComparison(depth, dataType); + case FUNCTION: + return generateFunction(depth + 1, MaterializeDataType.BOOLEAN); + case LIKE: + return new MaterializeLikeOperation(generateExpression(depth + 1, MaterializeDataType.TEXT), + generateExpression(depth + 1, MaterializeDataType.TEXT)); + case BETWEEN: + MaterializeDataType type = getMeaningfulType(); + return new MaterializeBetweenOperation(generateExpression(depth + 1, type), + generateExpression(depth + 1, type), generateExpression(depth + 1, type), Randomly.getBoolean()); + case POSIX_REGEX: + assert !expectedResult; + return new MaterializePOSIXRegularExpression(generateExpression(depth + 1, MaterializeDataType.TEXT), + generateExpression(depth + 1, MaterializeDataType.TEXT), POSIXRegex.getRandom()); + default: + throw new AssertionError(); + } + } + + private MaterializeDataType getMeaningfulType() { + // make it more likely that the expression does not only consist of constant + // expressions + if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { + return MaterializeDataType.getRandomType(); + } else { + return Randomly.fromList(columns).getType(); + } + } + + private MaterializeExpression generateFunction(int depth, MaterializeDataType type) { + if (MaterializeProvider.generateOnlyKnown || Randomly.getBoolean()) { + return generateFunctionWithKnownResult(depth, type); + } else { + return generateFunctionWithUnknownResult(depth, type); + } + } + + private MaterializeExpression generateComparison(int depth, MaterializeDataType dataType) { + MaterializeExpression leftExpr = generateExpression(depth + 1, dataType); + MaterializeExpression rightExpr = generateExpression(depth + 1, dataType); + return getComparison(leftExpr, rightExpr); + } + + private MaterializeExpression getComparison(MaterializeExpression leftExpr, MaterializeExpression rightExpr) { + return new MaterializeBinaryComparisonOperation(leftExpr, rightExpr, + MaterializeBinaryComparisonOperation.MaterializeBinaryComparisonOperator.getRandom()); + } + + private MaterializeExpression inOperation(int depth) { + MaterializeDataType type = MaterializeDataType.getRandomType(); + MaterializeExpression leftExpr = generateExpression(depth + 1, type); + List rightExpr = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + rightExpr.add(generateExpression(depth + 1, type)); + } + return new MaterializeInOperation(leftExpr, rightExpr, Randomly.getBoolean()); + } + + public static MaterializeExpression generateExpression(MaterializeGlobalState globalState, + MaterializeDataType type) { + return new MaterializeExpressionGenerator(globalState).generateExpression(0, type); + } + + public MaterializeExpression generateExpression(int depth, MaterializeDataType originalType) { + MaterializeDataType dataType = originalType; + if (dataType == MaterializeDataType.REAL && Randomly.getBoolean()) { + dataType = Randomly.fromOptions(MaterializeDataType.INT, MaterializeDataType.FLOAT); + } + if (dataType == MaterializeDataType.FLOAT && Randomly.getBoolean()) { + dataType = MaterializeDataType.INT; + } + if (!filterColumns(dataType).isEmpty() && Randomly.getBoolean()) { + return createColumnOfType(dataType); + } + return generateExpressionInternal(depth, dataType); + } + + private MaterializeExpression generateExpressionInternal(int depth, MaterializeDataType dataType) + throws AssertionError { + if (allowAggregateFunctions && Randomly.getBoolean()) { + allowAggregateFunctions = false; // aggregate function calls cannot be nested + return getAggregate(dataType); + } + if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { + // generic expression + if (Randomly.getBoolean() || depth > maxDepth) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(r, dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(r, dataType); + } else { + return createColumnOfType(dataType); + } + } + } else { + if (Randomly.getBoolean()) { + return new MaterializeCastOperation(generateExpression(depth + 1), getCompoundDataType(dataType)); + } else { + return generateFunctionWithUnknownResult(depth, dataType); + } + } + } else { + switch (dataType) { + case BOOLEAN: + return generateBooleanExpression(depth); + case INT: + return generateIntExpression(depth); + case TEXT: + return generateTextExpression(depth); + case DECIMAL: + case REAL: + case FLOAT: + return generateConstant(r, dataType); + case BIT: + return generateBitExpression(depth); + default: + throw new AssertionError(dataType); + } + } + } + + private static MaterializeCompoundDataType getCompoundDataType(MaterializeDataType type) { + switch (type) { + case BOOLEAN: + case DECIMAL: // TODO + case FLOAT: + case INT: + case REAL: + case BIT: + return MaterializeCompoundDataType.create(type); + case TEXT: // TODO + if (Randomly.getBoolean() || MaterializeProvider.generateOnlyKnown /* + * The PQS implementation does not check + * for size specifications + */) { + return MaterializeCompoundDataType.create(type); + } else { + return MaterializeCompoundDataType.create(type, (int) Randomly.getNotCachedInteger(1, 1000)); + } + default: + throw new AssertionError(type); + } + + } + + private enum TextExpression { + CAST, FUNCTION, CONCAT + } + + private MaterializeExpression generateTextExpression(int depth) { + TextExpression option; + List validOptions = new ArrayList<>(Arrays.asList(TextExpression.values())); + option = Randomly.fromList(validOptions); + + switch (option) { + case CAST: + return new MaterializeCastOperation(generateExpression(depth + 1), + getCompoundDataType(MaterializeDataType.TEXT)); + case FUNCTION: + return generateFunction(depth + 1, MaterializeDataType.TEXT); + case CONCAT: + return generateConcat(depth); + default: + throw new AssertionError(); + } + } + + private MaterializeExpression generateConcat(int depth) { + MaterializeExpression left = generateExpression(depth + 1, MaterializeDataType.TEXT); + MaterializeExpression right = generateExpression(depth + 1); + return new MaterializeConcatOperation(left, right); + } + + private enum BitExpression { + BINARY_OPERATION + }; + + private MaterializeExpression generateBitExpression(int depth) { + BitExpression option; + option = Randomly.fromOptions(BitExpression.values()); + switch (option) { + case BINARY_OPERATION: + return new MaterializeBinaryBitOperation(MaterializeBinaryBitOperator.getRandom(), + generateExpression(depth + 1, MaterializeDataType.BIT), + generateExpression(depth + 1, MaterializeDataType.BIT)); + default: + throw new AssertionError(); + } + } + + private enum IntExpression { + UNARY_OPERATION, FUNCTION, /* CAST, */BINARY_ARITHMETIC_EXPRESSION + } + + private MaterializeExpression generateIntExpression(int depth) { + IntExpression option; + option = Randomly.fromOptions(IntExpression.values()); + switch (option) { + case UNARY_OPERATION: + MaterializeExpression intExpression = generateExpression(depth + 1, MaterializeDataType.INT); + return new MaterializePrefixOperation(intExpression, + Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); + case FUNCTION: + return generateFunction(depth + 1, MaterializeDataType.INT); + case BINARY_ARITHMETIC_EXPRESSION: + return new MaterializeBinaryArithmeticOperation(generateExpression(depth + 1, MaterializeDataType.INT), + generateExpression(depth + 1, MaterializeDataType.INT), MaterializeBinaryOperator.getRandom()); + default: + throw new AssertionError(); + } + } + + private MaterializeExpression createColumnOfType(MaterializeDataType type) { + List columns = filterColumns(type); + MaterializeColumn fromList = Randomly.fromList(columns); + MaterializeConstant value = rw == null ? null : rw.getValues().get(fromList); + return MaterializeColumnValue.create(fromList, value); + } + + final List filterColumns(MaterializeDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + public MaterializeExpression generateExpressionWithExpectedResult(MaterializeDataType type) { + this.expectedResult = true; + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns) + .setRowValue(rw); + MaterializeExpression expr; + do { + expr = gen.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + public static MaterializeExpression generateConstant(Randomly r, MaterializeDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return MaterializeConstant.createNullConstant(); + } + switch (type) { + case INT: + if (Randomly.getBooleanWithSmallProbability()) { + return MaterializeConstant.createTextConstant(String.valueOf(r.getInteger())); + } else { + return MaterializeConstant.createIntConstant(r.getInteger()); + } + case BOOLEAN: + if (Randomly.getBooleanWithSmallProbability() && !MaterializeProvider.generateOnlyKnown) { + return MaterializeConstant + .createTextConstant(Randomly.fromOptions("TR", "TRUE", "FA", "FALSE", "0", "1", "ON", "off")); + } else { + return MaterializeConstant.createBooleanConstant(Randomly.getBoolean()); + } + case TEXT: + return MaterializeConstant.createTextConstant(r.getString()); + case DECIMAL: + return MaterializeConstant.createDecimalConstant(r.getRandomBigDecimal()); + case FLOAT: + return MaterializeConstant.createFloatConstant((float) r.getDouble()); + case REAL: + return MaterializeConstant.createDoubleConstant(r.getDouble()); + case BIT: + return MaterializeConstant.createBitConstant(r.getInteger()); + default: + throw new AssertionError(type); + } + } + + public static MaterializeExpression generateExpression(MaterializeGlobalState globalState, + List columns, MaterializeDataType type) { + return new MaterializeExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); + } + + public static MaterializeExpression generateExpression(MaterializeGlobalState globalState, + List columns) { + return new MaterializeExpressionGenerator(globalState).setColumns(columns).generateExpression(0); + + } + + public List generateExpressions(int nr) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(0)); + } + return expressions; + } + + public MaterializeExpression generateExpression(MaterializeDataType dataType) { + return generateExpression(0, dataType); + } + + public MaterializeExpressionGenerator setGlobalState(MaterializeGlobalState globalState) { + this.globalState = globalState; + return this; + } + + public MaterializeExpression generateHavingClause() { + this.allowAggregateFunctions = true; + MaterializeExpression expression = generateExpression(MaterializeDataType.BOOLEAN); + this.allowAggregateFunctions = false; + return expression; + } + + public MaterializeExpression generateAggregate() { + return getAggregate(MaterializeDataType.getRandomType()); + } + + private MaterializeExpression getAggregate(MaterializeDataType dataType) { + List aggregates = MaterializeAggregateFunction.getAggregates(dataType); + MaterializeAggregateFunction agg = Randomly.fromList(aggregates); + return generateArgsForAggregate(dataType, agg); + } + + public MaterializeAggregate generateArgsForAggregate(MaterializeDataType dataType, + MaterializeAggregateFunction agg) { + List types = agg.getTypes(dataType); + List args = new ArrayList<>(); + for (MaterializeDataType argType : types) { + args.add(generateExpression(argType)); + } + return new MaterializeAggregate(args, agg); + } + + public MaterializeExpressionGenerator allowAggregates(boolean value) { + allowAggregateFunctions = value; + return this; + } + + @Override + public MaterializeExpression generatePredicate() { + return generateExpression(MaterializeDataType.BOOLEAN); + } + + @Override + public MaterializeExpression negatePredicate(MaterializeExpression predicate) { + return new MaterializePrefixOperation(predicate, MaterializePrefixOperation.PrefixOperator.NOT); + } + + @Override + public MaterializeExpression isNull(MaterializeExpression expr) { + return new MaterializePostfixOperation(expr, PostfixOperator.IS_NULL); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java new file mode 100644 index 000000000..9d7a91b50 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java @@ -0,0 +1,80 @@ +package sqlancer.materialize.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; + +public final class MaterializeIndexGenerator { + + private MaterializeIndexGenerator() { + } + + public enum IndexType { + BTREE, HASH, GIST, GIN + } + + public static SQLQueryAdapter generate(MaterializeGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE"); + sb.append(" INDEX "); + MaterializeTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized + // views + sb.append(MaterializeCommon.getFreeIndexName(globalState.getSchema())); + sb.append(" ON "); + sb.append(randomTable.getName()); + IndexType method; + method = IndexType.BTREE; + + sb.append("("); + if (method == IndexType.HASH) { + sb.append(randomTable.getRandomColumn().getName()); + } else { + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(randomTable.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NULLS "); + sb.append(Randomly.fromOptions("FIRST", "LAST")); + } + } + } + + sb.append(")"); + errors.add("already contains data"); // CONCURRENT INDEX failed + errors.add("You might need to add explicit type casts"); + errors.add(" collations are not supported"); + errors.add("because it has pending trigger events"); + errors.add("could not determine which collation to use for index expression"); + errors.add("could not determine which collation to use for string comparison"); + errors.add("is duplicated"); + errors.add("access method \"gin\" does not support unique indexes"); + errors.add("access method \"hash\" does not support unique indexes"); + errors.add("already exists"); + errors.add("could not create unique index"); + errors.add("has no default operator class"); + errors.add("does not support"); + errors.add("does not support casting"); + errors.add("unsupported UNIQUE constraint with partition key definition"); + errors.add("insufficient columns in UNIQUE constraint definition"); + errors.add("invalid input syntax for"); + errors.add("must be type "); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("out of range"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("functions in index expression must be marked IMMUTABLE"); + errors.add("result of range difference would not be contiguous"); + errors.add("which is part of the partition key"); + MaterializeCommon.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java new file mode 100644 index 000000000..01015514b --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java @@ -0,0 +1,106 @@ +package sqlancer.materialize.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeExpression; + +public final class MaterializeInsertGenerator { + + private MaterializeInsertGenerator() { + } + + public static SQLQueryAdapter insert(MaterializeGlobalState globalState) { + MaterializeTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + ExpectedErrors errors = new ExpectedErrors(); + errors.add("cannot insert into column"); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonInsertUpdateErrors(errors); + MaterializeCommon.addCommonExpressionErrors(errors); + errors.add("multiple assignments to same column"); + errors.add("violates foreign key constraint"); + errors.add("value too long for type character varying"); + errors.add("conflicting key value violates exclusion constraint"); + errors.add("violates not-null constraint"); + errors.add("current transaction is aborted"); + errors.add("bit string too long"); + errors.add("new row violates check option for view"); + errors.add("reached maximum value of sequence"); + errors.add("but expression is of type"); + StringBuilder sb = new StringBuilder(); + sb.append("INSERT INTO "); + sb.append(table.getName()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES"); + + if (globalState.getDbmsSpecificOptions().allowBulkInsert && Randomly.getBooleanWithSmallProbability()) { + StringBuilder sbRowValue = new StringBuilder(); + sbRowValue.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sbRowValue.append(", "); + } + sbRowValue.append(MaterializeVisitor.asString(MaterializeExpressionGenerator + .generateConstant(globalState.getRandomly(), columns.get(i).getType()))); + } + sbRowValue.append(")"); + + int n = (int) Randomly.getNotCachedInteger(100, 1000); + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(sbRowValue); + } + } else { + int n = Randomly.smallNumber() + 1; + for (int i = 0; i < n; i++) { + if (i != 0) { + sb.append(", "); + } + insertRow(globalState, sb, columns); + } + } + errors.add("duplicate key value violates unique constraint"); + errors.add("identity column defined as GENERATED ALWAYS"); + errors.add("out of range"); + errors.add("violates check constraint"); + errors.add("no partition of relation"); + errors.add("invalid input syntax"); + errors.add("division by zero"); + errors.add("violates foreign key constraint"); + errors.add("data type unknown"); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private static void insertRow(MaterializeGlobalState globalState, StringBuilder sb, + List columns) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + MaterializeExpression generateConstant; + if (Randomly.getBoolean()) { + generateConstant = MaterializeExpressionGenerator.generateConstant(globalState.getRandomly(), + columns.get(i).getType()); + } else { + generateConstant = new MaterializeExpressionGenerator(globalState) + .generateExpression(columns.get(i).getType()); + } + sb.append(MaterializeVisitor.asString(generateConstant)); + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java new file mode 100644 index 000000000..16f4f1be8 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java @@ -0,0 +1,64 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.ForClause; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.SelectType; + +public final class MaterializeRandomQueryGenerator { + + private MaterializeRandomQueryGenerator() { + } + + public static MaterializeSelect createRandomQuery(int nrColumns, MaterializeGlobalState globalState) { + List columns = new ArrayList<>(); + MaterializeTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState) + .setColumns(tables.getColumns()); + for (int i = 0; i < nrColumns; i++) { + columns.add(gen.generateExpression(0)); + } + MaterializeSelect select = new MaterializeSelect(); + select.setSelectType(SelectType.getRandom()); + if (select.getSelectOption() == SelectType.DISTINCT && Randomly.getBoolean()) { + select.setDistinctOnClause(gen.generateExpression(0)); + } + select.setFromList(tables.getTables().stream().map(t -> new MaterializeFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return select; + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeTableGenerator.java b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java new file mode 100644 index 000000000..f132f7370 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java @@ -0,0 +1,127 @@ +package sqlancer.materialize.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; + +public class MaterializeTableGenerator { + + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + @SuppressWarnings("unused") + private boolean isTemporaryTable; + @SuppressWarnings("unused") + private final MaterializeSchema newSchema; + private final List columnsToBeAdded = new ArrayList<>(); + protected final ExpectedErrors errors = new ExpectedErrors(); + private final MaterializeTable table; + private final boolean generateOnlyKnown; + private final MaterializeGlobalState globalState; + + public MaterializeTableGenerator(String tableName, MaterializeSchema newSchema, boolean generateOnlyKnown, + MaterializeGlobalState globalState) { + this.tableName = tableName; + this.newSchema = newSchema; + this.generateOnlyKnown = generateOnlyKnown; + this.globalState = globalState; + table = new MaterializeTable(tableName, columnsToBeAdded, null, null, null, false, false); + errors.add("invalid input syntax for"); + errors.add("is not unique"); + errors.add("integer out of range"); + errors.add("division by zero"); + errors.add("cannot create partitioned table as inheritance child"); + errors.add("does not support casting"); + errors.add("ERROR: functions in index expression must be marked IMMUTABLE"); + errors.add("functions in partition key expression must be marked IMMUTABLE"); + errors.add("functions in index predicate must be marked IMMUTABLE"); + errors.add("has no default operator class for access method"); + errors.add("does not exist for access method"); + errors.add("does not accept data type"); + errors.add("but default expression is of type text"); + errors.add("has pseudo-type unknown"); + errors.add("no collation was derived for partition key column"); + errors.add("inherits from generated column but specifies identity"); + errors.add("inherits from generated column but specifies default"); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonTableErrors(errors); + } + + public static SQLQueryAdapter generate(String tableName, MaterializeSchema newSchema, boolean generateOnlyKnown, + MaterializeGlobalState globalState) { + return new MaterializeTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); + } + + protected SQLQueryAdapter generate() { + sb.append("CREATE"); + sb.append(" TABLE"); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS"); + } + sb.append(" "); + sb.append(tableName); + createStandard(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void createStandard() throws AssertionError { + sb.append("("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + String name = DBMSCommon.createColumnName(i); + createColumn(name); + } + sb.append(")"); + } + + private void createColumn(String name) throws AssertionError { + sb.append(name); + sb.append(" "); + MaterializeDataType type = MaterializeDataType.getRandomType(); + MaterializeCommon.appendDataType(type, sb, true, generateOnlyKnown, globalState.getCollates()); + MaterializeColumn c = new MaterializeColumn(name, type); + c.setTable(table); + columnsToBeAdded.add(c); + sb.append(" "); + if (Randomly.getBoolean()) { + createColumnConstraint(type); + } + } + + private enum ColumnConstraint { + DEFAULT + }; + + private void createColumnConstraint(MaterializeDataType type) { + List constraintSubset = Randomly.nonEmptySubset(ColumnConstraint.values()); + for (ColumnConstraint c : constraintSubset) { + sb.append(" "); + switch (c) { + case DEFAULT: + sb.append("DEFAULT"); + sb.append(" ("); + sb.append(MaterializeVisitor + .asString(MaterializeExpressionGenerator.generateExpression(globalState, type))); + sb.append(")"); + // CREATE TEMPORARY TABLE t1(c0 smallint DEFAULT ('566963878')); + errors.add("out of range"); + errors.add("is a generated column"); + break; + default: + throw new AssertionError(sb); + } + } + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java new file mode 100644 index 000000000..1b7e69208 --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java @@ -0,0 +1,78 @@ +package sqlancer.materialize.gen; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeExpression; + +public final class MaterializeUpdateGenerator extends AbstractUpdateGenerator { + + private final MaterializeGlobalState globalState; + private MaterializeTable randomTable; + + private MaterializeUpdateGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; + errors.addAll(Arrays.asList("conflicting key value violates exclusion constraint", + "reached maximum value of sequence", "violates foreign key constraint", "violates not-null constraint", + "violates unique constraint", "out of range", "does not support casting", "must be type boolean", + "is not unique", " bit string too long", "can only be updated to DEFAULT", "division by zero", + "You might need to add explicit type casts.", "invalid regular expression", + "View columns that are not columns of their base relation are not updatable")); + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + return new MaterializeUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + randomTable = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + List columns = randomTable.getRandomNonEmptyColumnSubset(); + sb.append("UPDATE "); + sb.append(randomTable.getName()); + sb.append(" SET "); + errors.add("multiple assignments to same column"); // view whose columns refer to a column in the referenced + // table multiple times + errors.add("new row violates check option for view"); + MaterializeCommon.addCommonInsertUpdateErrors(errors); + updateColumns(columns); + errors.add("invalid input syntax for "); + errors.add("operator does not exist: text = boolean"); + errors.add("violates check constraint"); + errors.add("could not determine which collation to use for string comparison"); + errors.add("but expression is of type"); + MaterializeCommon.addCommonExpressionErrors(errors); + if (!Randomly.getBooleanWithSmallProbability()) { + sb.append(" WHERE "); + MaterializeExpression where = MaterializeExpressionGenerator.generateExpression(globalState, + randomTable.getColumns(), MaterializeDataType.BOOLEAN); + sb.append(MaterializeVisitor.asString(where)); + } + + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + @Override + protected void updateValue(MaterializeColumn column) { + if (!Randomly.getBoolean()) { + MaterializeExpression constant = MaterializeExpressionGenerator.generateConstant(globalState.getRandomly(), + column.getType()); + sb.append(MaterializeVisitor.asString(constant)); + } else { + sb.append("("); + MaterializeExpression expr = MaterializeExpressionGenerator.generateExpression(globalState, + randomTable.getColumns(), column.getType()); + // caused by casts + sb.append(MaterializeVisitor.asString(expr)); + sb.append(")"); + } + } + +} diff --git a/src/sqlancer/materialize/gen/MaterializeViewGenerator.java b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java new file mode 100644 index 000000000..c8cd0e93f --- /dev/null +++ b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java @@ -0,0 +1,73 @@ +package sqlancer.materialize.gen; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeSelect; + +public final class MaterializeViewGenerator { + + private MaterializeViewGenerator() { + } + + public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("CREATE"); + @SuppressWarnings("unused") + boolean materialized; + @SuppressWarnings("unused") + boolean recursive = false; + if (Randomly.getBoolean()) { + sb.append(" MATERIALIZED"); + materialized = true; + } else { + if (Randomly.getBoolean()) { + sb.append(" OR REPLACE"); + } + materialized = false; + } + sb.append(" VIEW "); + int i = 0; + String[] name = new String[1]; + while (true) { + name[0] = "v" + i++; + if (globalState.getSchema().getDatabaseTables().stream() + .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { + break; + } + } + sb.append(name[0]); + sb.append("("); + int nrColumns = Randomly.smallNumber() + 1; + for (i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(DBMSCommon.createColumnName(i)); + } + sb.append(")"); + sb.append(" AS ("); + MaterializeSelect select = MaterializeRandomQueryGenerator.createRandomQuery(nrColumns, globalState); + sb.append(MaterializeVisitor.asString(select)); + sb.append(")"); + MaterializeCommon.addGroupingErrors(errors); + errors.add("already exists"); + errors.add("cannot drop columns from view"); + errors.add("non-integer constant in ORDER BY"); // TODO + errors.add("for SELECT DISTINCT, ORDER BY expressions must appear in select list"); // TODO + errors.add("cannot change data type of view column"); + errors.add("specified more than once"); // TODO + errors.add("materialized views must not use temporary tables or views"); + errors.add("does not have the form non-recursive-term UNION [ALL] recursive-term"); + errors.add("is not a view"); + errors.add("non-integer constant in DISTINCT ON"); + errors.add("unable to parse column reference in DISTINCT ON clause"); + errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + MaterializeCommon.addCommonExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java new file mode 100644 index 000000000..fbfbcb132 --- /dev/null +++ b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java @@ -0,0 +1,176 @@ +package sqlancer.materialize.oracle; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeCastOperation; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; +import sqlancer.materialize.ast.MaterializePostfixText; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSelect.SelectType; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; +import sqlancer.materialize.oracle.tlp.MaterializeTLPBase; + +public class MaterializeNoRECOracle extends NoRECBase + implements TestOracle { + + private final MaterializeSchema s; + + public MaterializeNoRECOracle(MaterializeGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonFetchErrors(errors); + } + + @Override + public void check() throws SQLException { + MaterializeTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + MaterializeExpression randomWhereCondition = getRandomWhereCondition(columns); + List tables = randomTables.getTables(); + + List joinStatements = getJoinStatements(state, columns, tables); + List fromTables = tables.stream() + .map(t -> new MaterializeFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); + int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, + firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + public static List getJoinStatements(MaterializeGlobalState globalState, + List columns, List tables) { + List joinStatements = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeTable table = Randomly.fromList(tables); + tables.remove(table); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, + options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + private MaterializeExpression getRandomWhereCondition(List columns) { + return new MaterializeExpressionGenerator(state).setColumns(columns) + .generateExpression(MaterializeDataType.BOOLEAN); + } + + private int getUnoptimizedQueryCount(List fromTables, + MaterializeExpression randomWhereCondition, List joinStatements) throws SQLException { + MaterializeSelect select = new MaterializeSelect(); + MaterializeCastOperation isTrue = new MaterializeCastOperation(randomWhereCondition, + MaterializeCompoundDataType.create(MaterializeDataType.INT)); + MaterializePostfixText asText = new MaterializePostfixText(isTrue, " as count", null, MaterializeDataType.INT); + select.setFetchColumns(Arrays.asList(asText)); + select.setFromList(fromTables); + select.setSelectType(SelectType.ALL); + select.setJoinClauses(joinStatements); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + MaterializeVisitor.asString(select) + ") as res"; + if (options.logEachSelect()) { + logger.writeCurrent(unoptimizedQueryString); + } + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGet(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getOptimizedQueryCount(List randomTables, List columns, + MaterializeExpression randomWhereCondition, List joinStatements) throws SQLException { + MaterializeSelect select = new MaterializeSelect(); + MaterializeColumnValue allColumns = new MaterializeColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(Arrays.asList(allColumns)); + select.setFromList(randomTables); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions( + new MaterializeExpressionGenerator(state).setColumns(columns).generateOrderBy()); + } + select.setSelectType(SelectType.ALL); + select.setJoinClauses(joinStatements); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = MaterializeVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + + @Override + public String getLastQueryString() { + return optimizedQueryString; + } +} diff --git a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..07bfb470f --- /dev/null +++ b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java @@ -0,0 +1,151 @@ +package sqlancer.materialize.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeRowValue; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; + +public class MaterializePivotedQuerySynthesisOracle extends + PivotedQuerySynthesisBase { + + private List fetchColumns; + + public MaterializePivotedQuerySynthesisOracle(MaterializeGlobalState globalState) throws SQLException { + super(globalState); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonFetchErrors(errors); + } + + @Override + public SQLQueryAdapter getRectifiedQuery() throws SQLException { + MaterializeTables randomFromTables = globalState.getSchema().getRandomTableNonEmptyTables(); + + MaterializeSelect selectStatement = new MaterializeSelect(); + selectStatement.setSelectType(Randomly.fromOptions(MaterializeSelect.SelectType.values())); + List columns = randomFromTables.getColumns(); + pivotRow = randomFromTables.getRandomRowValue(globalState.getConnection()); + + fetchColumns = columns; + selectStatement.setFromList(randomFromTables.getTables().stream().map(t -> new MaterializeFromTable(t, false)) + .collect(Collectors.toList())); + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new MaterializeColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + MaterializeExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + MaterializeExpression limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + MaterializeExpression offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + List orderBy = new MaterializeExpressionGenerator(globalState).setColumns(columns) + .generateOrderBy(); + selectStatement.setOrderByExpressions(orderBy); + return new SQLQueryAdapter(MaterializeVisitor.asString(selectStatement)); + } + + /* + * Prevent name collisions by aliasing the column. + */ + private MaterializeColumn getFetchValueAliasedColumn(MaterializeColumn c) { + MaterializeColumn aliasedColumn = new MaterializeColumn( + c.getName() + " AS " + c.getTable().getName() + c.getName(), c.getType()); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + private List generateGroupByClause(List columns, MaterializeRowValue rw) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> MaterializeColumnValue.create(c, rw.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private MaterializeConstant generateLimit() { + if (Randomly.getBoolean()) { + return MaterializeConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private MaterializeExpression generateOffset() { + if (Randomly.getBoolean()) { + return MaterializeConstant.createIntConstant(0); + } else { + return null; + } + } + + private MaterializeExpression generateRectifiedExpression(List columns, MaterializeRowValue rw) { + MaterializeExpression expr = new MaterializeExpressionGenerator(globalState).setColumns(columns).setRowValue(rw) + .generateExpressionWithExpectedResult(MaterializeDataType.BOOLEAN); + MaterializeExpression result; + if (expr.getExpectedValue().isNull()) { + result = MaterializePostfixOperation.create(expr, PostfixOperator.IS_NULL); + } else { + result = MaterializePostfixOperation.create(expr, + expr.getExpectedValue().cast(MaterializeDataType.BOOLEAN).asBoolean() ? PostfixOperator.IS_TRUE + : PostfixOperator.IS_FALSE); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query query) throws SQLException { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); // ANOTHER SELECT TO USE ORDER BY without restrictions + sb.append(query.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (MaterializeColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).getTextRepresentation()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + @Override + protected String getExpectedValues(MaterializeExpression expr) { + return MaterializeVisitor.asExpectedValues(expr); + } + +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java new file mode 100644 index 000000000..4f5f67294 --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java @@ -0,0 +1,187 @@ +package sqlancer.materialize.oracle.tlp; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import org.postgresql.util.PSQLException; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeAggregate; +import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; +import sqlancer.materialize.ast.MaterializeAlias; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializePostfixOperation; +import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializePrefixOperation; +import sqlancer.materialize.ast.MaterializePrefixOperation.PrefixOperator; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.gen.MaterializeCommon; + +public class MaterializeTLPAggregateOracle extends MaterializeTLPBase implements TestOracle { + private String generatedQueryString; + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public MaterializeTLPAggregateOracle(MaterializeGlobalState state) { + super(state); + MaterializeCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + aggregateCheck(); + } + + protected void aggregateCheck() throws SQLException { + MaterializeAggregateFunction aggregateFunction = Randomly.fromOptions(MaterializeAggregateFunction.MAX, + MaterializeAggregateFunction.MIN, MaterializeAggregateFunction.SUM, + MaterializeAggregateFunction.BIT_AND, MaterializeAggregateFunction.BIT_OR, + MaterializeAggregateFunction.BOOL_AND, MaterializeAggregateFunction.BOOL_OR, + MaterializeAggregateFunction.COUNT); + MaterializeAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), + aggregateFunction); + List fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add(gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + originalQuery = MaterializeVisitor.asString(select); + generatedQueryString = originalQuery; + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + String queryFormatString = "-- %s;\n-- result: %s"; + String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); + String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); + state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); + if (firstResult == null && secondResult != null || firstResult != null && secondResult == null + || firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult != null && secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + String assertionMessage = String.format("the results mismatch!\n%s\n%s", firstQueryString, + secondQueryString); + throw new AssertionError(assertionMessage); + } + } + + private String createMetamorphicUnionQuery(MaterializeSelect select, MaterializeAggregate aggregate, + List from) { + String metamorphicQuery; + MaterializeExpression whereClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeExpression negatedClause = new MaterializePrefixOperation(whereClause, PrefixOperator.NOT); + MaterializeExpression notNullClause = new MaterializePostfixOperation(whereClause, PostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); + MaterializeSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); + MaterializeSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); + MaterializeSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += MaterializeVisitor.asString(leftSelect) + " UNION ALL " + + MaterializeVisitor.asString(middleSelect) + " UNION ALL " + MaterializeVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + // log TLP Aggregate SELECT queries on the current log file + if (state.getOptions().logEachSelect()) { + // TODO: refactor me + state.getLogger().writeCurrent(queryString); + try { + state.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + } catch (PSQLException e) { + throw new AssertionError(queryString, e); + } + return resultString; + } + + private List mapped(MaterializeAggregate aggregate) { + switch (aggregate.getFunction()) { + case SUM: + case COUNT: + case BIT_AND: + case BIT_OR: + case BOOL_AND: + case BOOL_OR: + case MAX: + case MIN: + return aliasArgs(Arrays.asList(aggregate)); + default: + throw new AssertionError(aggregate.getFunction()); + } + } + + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); + int i = 0; + for (MaterializeExpression expr : originalAggregateArgs) { + args.add(new MaterializeAlias(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(MaterializeAggregate aggregate) { + switch (aggregate.getFunction()) { + case COUNT: + return MaterializeAggregateFunction.SUM.toString() + "(agg0)"; + default: + return aggregate.getFunction().toString() + "(agg0)"; + } + } + + private MaterializeSelect getSelect(List aggregates, List from, + MaterializeExpression whereClause, List joinList) { + MaterializeSelect leftSelect = new MaterializeSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinClauses(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java new file mode 100644 index 000000000..4aa2196d1 --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java @@ -0,0 +1,123 @@ +package sqlancer.materialize.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; +import sqlancer.materialize.ast.MaterializeColumnValue; +import sqlancer.materialize.ast.MaterializeConstant; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.ForClause; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; +import sqlancer.materialize.oracle.MaterializeNoRECOracle; + +public class MaterializeTLPBase + extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + protected MaterializeSchema s; + protected MaterializeTables targetTables; + protected MaterializeExpressionGenerator gen; + protected MaterializeSelect select; + + public MaterializeTLPBase(MaterializeGlobalState state) { + super(state); + MaterializeCommon.addCommonExpressionErrors(errors); + MaterializeCommon.addCommonFetchErrors(errors); + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + List tables = targetTables.getTables(); + List joins = getJoinStatements(state, targetTables.getColumns(), tables); + generateSelectBase(tables, joins); + } + + protected List getJoinStatements(MaterializeGlobalState globalState, + List columns, List tables) { + return MaterializeNoRECOracle.getJoinStatements(state, columns, tables); + // TODO joins + } + + protected void generateSelectBase(List tables, List joins) { + List tableList = tables.stream() + .map(t -> new MaterializeFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); + gen = new MaterializeExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new MaterializeSelect(); + select.setFetchColumns(generateFetchColumns()); + select.setFromList(tableList); + select.setWhereClause(null); + select.setJoinClauses(joins); + if (Randomly.getBoolean()) { + select.setForClause(ForClause.getRandom()); + } + } + + List generateFetchColumns() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new MaterializeColumnValue(MaterializeColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(targetTables.getColumns()); + for (MaterializeColumn c : targetColumns) { + fetchColumns.add(new MaterializeColumnValue(c, null)); + } + return fetchColumns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + + public static MaterializeSubquery createSubquery(MaterializeGlobalState globalState, String name, + MaterializeTables tables) { + List columns = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState) + .setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + MaterializeSelect select = new MaterializeSelect(); + select.setFromList(tables.getTables().stream().map(t -> new MaterializeFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return new MaterializeSubquery(select, name); + } +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java new file mode 100644 index 000000000..46d6c6d74 --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java @@ -0,0 +1,83 @@ +package sqlancer.materialize.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.gen.MaterializeCommon; + +public class MaterializeTLPHavingOracle extends MaterializeTLPBase { + private String generatedQueryString; + + public MaterializeTLPHavingOracle(MaterializeGlobalState state) { + super(state); + MaterializeCommon.addGroupingErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + havingCheck(); + } + + protected void havingCheck() throws SQLException { + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(MaterializeDataType.BOOLEAN)); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = MaterializeVisitor.asString(select); + generatedQueryString = originalQueryString; + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + // See https://github.com/MaterializeInc/materialize/issues/18346, have to check if predicate errors by putting + // it in SELECT first + List originalColumns = select.getFetchColumns(); + List checkColumns = new ArrayList<>(); + checkColumns.add(predicate); + select.setFetchColumns(checkColumns); + String errorCheckQueryString = MaterializeVisitor.asString(select); + ComparatorHelper.getResultSetFirstColumnAsString(errorCheckQueryString, errors, state); + select.setFetchColumns(originalColumns); + + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + select.setHavingClause(predicate); + String firstQueryString = MaterializeVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = MaterializeVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = MaterializeVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + protected MaterializeExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List generateFetchColumns() { + List expressions = gen.allowAggregates(true) + .generateExpressions(Randomly.smallNumber() + 1); + gen.allowAggregates(false); + return expressions; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java new file mode 100644 index 000000000..6af0ac2cc --- /dev/null +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java @@ -0,0 +1,52 @@ +package sqlancer.materialize.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeVisitor; + +public class MaterializeTLPWhereOracle extends MaterializeTLPBase { + private String generatedQueryString; + + public MaterializeTLPWhereOracle(MaterializeGlobalState state) { + super(state); + } + + @Override + public void check() throws SQLException { + super.check(); + whereCheck(); + } + + protected void whereCheck() throws SQLException { + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBy()); + } + String originalQueryString = MaterializeVisitor.asString(select); + generatedQueryString = originalQueryString; + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setOrderByExpressions(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = MaterializeVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = MaterializeVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = MaterializeVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/test/sqlancer/dbms/TestMaterialize.java b/test/sqlancer/dbms/TestMaterialize.java new file mode 100644 index 000000000..1818aa513 --- /dev/null +++ b/test/sqlancer/dbms/TestMaterialize.java @@ -0,0 +1,24 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMaterialize { + + String materializeAvailable = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materializeAvailable != null && materializeAvailable.equalsIgnoreCase("true"); + + @Test + public void test() { + assumeTrue(materializeIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--username", "materialize", + "materialize", "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/dbms/TestMaterializePQS.java b/test/sqlancer/dbms/TestMaterializePQS.java new file mode 100644 index 000000000..5d4d795e4 --- /dev/null +++ b/test/sqlancer/dbms/TestMaterializePQS.java @@ -0,0 +1,25 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMaterializePQS { + + String materializeAvailable = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materializeAvailable != null && materializeAvailable.equalsIgnoreCase("true"); + + @Test + public void test() { + assumeTrue(materializeIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "--username", "materialize", "materialize", "--oracle", "pqs", + "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQPG.java b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java new file mode 100644 index 000000000..0564d048c --- /dev/null +++ b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java @@ -0,0 +1,23 @@ +package sqlancer.qpg.cockroachdb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestMaterializeQPG { + + @Test + public void testMaterializeQPG() { + String materialize = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materialize != null && materialize.equalsIgnoreCase("true"); + assumeTrue(materializeIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "--username", + "materialize", "materialize", "--oracle", "QUERY_PARTITIONING", "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java new file mode 100644 index 000000000..08e402037 --- /dev/null +++ b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java @@ -0,0 +1,49 @@ +package sqlancer.qpg.materialize; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.materialize.MaterializeOptions; +import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeProvider; + +public class TestMaterializeQueryPlan { + + @Test + void testMaterializeQueryPlan() throws Exception { + String materialize = System.getenv("MATERIALIZE_AVAILABLE"); + boolean materializeIsAvailable = materialize != null && materialize.equalsIgnoreCase("true"); + assumeTrue(materializeIsAvailable); + + String databaseName = "materialize"; + MaterializeProvider provider = new MaterializeProvider(); + MaterializeGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + MaterializeOptions materializeOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(materializeOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + + assertEquals( + "Return;Union;Get l0;Project (#2, #3, #0);Union;Negate;Project (#2);Get materialize.public.t2;Get materialize.public.t2;With;Get materialize.public.t1;Get materialize.public.t2;;Source materialize.public.t1;", + queryPlan); + } + +} From 63b03fb1e22be6c7e1b2a411da2ebbbeab9bfc61 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 6 Apr 2023 14:11:57 +0200 Subject: [PATCH 336/856] Mz: Fix typo in expected error (#775) Caused by: org.postgresql.util.PSQLException: ERROR: result exceeds max size of 1073.7 MB --- src/sqlancer/materialize/gen/MaterializeCommon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/gen/MaterializeCommon.java b/src/sqlancer/materialize/gen/MaterializeCommon.java index 0c4847bed..7d722e102 100644 --- a/src/sqlancer/materialize/gen/MaterializeCommon.java +++ b/src/sqlancer/materialize/gen/MaterializeCommon.java @@ -31,7 +31,7 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("non-integer constant in GROUP BY"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY position"); - errors.add("results exceed max size of"); + errors.add("results exceeds max size of"); errors.add("does not exist"); errors.add("aggregate functions are not allowed in"); From 28e6c15086c3e81b93dc2a3980c957932cd383d7 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 12 Apr 2023 09:18:51 +0200 Subject: [PATCH 337/856] Materialize: Use Serializable isolation level (#778) --- src/sqlancer/materialize/MaterializeProvider.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java index dc802d2aa..3e6e1c59f 100644 --- a/src/sqlancer/materialize/MaterializeProvider.java +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -216,6 +216,12 @@ public SQLConnection createDatabase(MaterializeGlobalState globalState) throws S globalState.getState().logStatement(String.format("\\c %s;", databaseName)); con = DriverManager.getConnection("jdbc:" + testURL, username, password); + // Serializable transaction isolation is much faster than Strict + // Serializable and should guarantee enough for SQLancer: + // https://materialize.com/docs/overview/isolation-level/ + try (Statement s = con.createStatement()) { + s.execute("SET transaction_isolation = 'SERIALIZABLE'"); + } return new SQLConnection(con); } From ba2059740dcc6afe65cbd49d66f29e9afe4c4f15 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 13 Apr 2023 23:00:41 +0800 Subject: [PATCH 338/856] tidb: support tidb_enable_non_prepared_plan_cache (#776) * tidb: support tidb_enable_non_prepared_plan_cache Signed-off-by: Weizhen Wang * update Signed-off-by: Weizhen Wang --------- Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBOptions.java | 3 +++ src/sqlancer/tidb/TiDBProvider.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 764c4bfd9..6fbab6ba8 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -32,6 +32,9 @@ public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracle = Arrays.asList(TiDBOracleFactory.QUERY_PARTITIONING); + @Parameter(names = "--enable-non-prepared-plan-cache") + public boolean nonPreparePlanCache; + public enum TiDBOracleFactory implements OracleFactory { HAVING { @Override diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index c4f13a207..91223a246 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -156,6 +156,9 @@ public SQLConnection createDatabase(TiDBGlobalState globalState) throws SQLExcep globalState.getState().logStatement("USE " + databaseName); try (Statement s = con.createStatement()) { s.execute("DROP DATABASE IF EXISTS " + databaseName); + if (globalState.getDbmsSpecificOptions().nonPreparePlanCache) { + s.execute("set global tidb_enable_non_prepared_plan_cache=ON;"); + } } try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); From b4cd36e5c47242837a8ee41f558e74bef99cb369 Mon Sep 17 00:00:00 2001 From: Rachael Harding Date: Wed, 26 Apr 2023 08:39:00 -0700 Subject: [PATCH 339/856] Update CockroachDB to v23.1.0-beta.2 (#779) Disable ignored errors from CockroachDB bugs that have been fixed. --- .github/workflows/main.yml | 12 +++++----- src/sqlancer/cockroachdb/CockroachDBBugs.java | 24 ++++++++++--------- .../cockroachdb/CockroachDBErrors.java | 4 ++++ .../gen/CockroachDBTableGenerator.java | 1 + 4 files changed, 24 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1892285b2..d4166e941 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -122,11 +122,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.3.linux-amd64.tgz | tar xvz - cd cockroach-v22.2.3.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-beta.2.linux-amd64.tgz | tar xvz + cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -145,11 +145,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v22.2.3.linux-amd64.tgz | tar xvz - cd cockroach-v22.2.3.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-beta.2.linux-amd64.tgz | tar xvz + cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v22.2.3.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 0d708c9a2..55045f637 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -18,41 +18,43 @@ public final class CockroachDBBugs { public static boolean bug83874 = true; // https://github.com/cockroachdb/cockroach/issues/83973 - public static boolean bug83973 = true; + public static boolean bug83973; // https://github.com/cockroachdb/cockroach/issues/83976 - public static boolean bug83976 = true; + public static boolean bug83976; + // The following bug is closed, but leave it enabled until + // the underlying interval issue is resolved. + // https://github.com/cockroachdb/cockroach/issues/84078 // https://github.com/cockroachdb/cockroach/issues/84154 public static boolean bug84154 = true; // https://github.com/cockroachdb/cockroach/issues/85356 - public static boolean bug85356 = true; + public static boolean bug85356; // https://github.com/cockroachdb/cockroach/issues/85371 - public static boolean bug85371 = true; + public static boolean bug85371; // https://github.com/cockroachdb/cockroach/issues/85389 - public static boolean bug85389 = true; + public static boolean bug85389; // https://github.com/cockroachdb/cockroach/issues/85390 - public static boolean bug85390 = true; + public static boolean bug85390; // https://github.com/cockroachdb/cockroach/issues/85393 - public static boolean bug85393 = true; + public static boolean bug85393; // https://github.com/cockroachdb/cockroach/issues/85394 public static boolean bug85394 = true; // https://github.com/cockroachdb/cockroach/issues/85441 - public static boolean bug85441 = true; + public static boolean bug85441; // https://github.com/cockroachdb/cockroach/issues/85499 - public static boolean bug85499 = true; + public static boolean bug85499; // https://github.com/cockroachdb/cockroach/issues/88037 - // TODO: This should be fixed in v22.2.1. - public static boolean bug88037 = true; + public static boolean bug88037; private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index bbbda7baf..e282981dc 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -74,6 +74,10 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add(" unsupported comparison operator: NOT LIKE != "); errors.add("expected DEFAULT expression to have type bytes"); + errors.add("expected DEFAULT (in CREATE TABLE) expression to have type bytes"); + errors.add("expected DEFAULT (in CREATE VIEW) expression to have type bytes"); + errors.add("expected DEFAULT (in SET DEFAULT) expression to have type bytes"); + errors.add("expected DEFAULT (in ADD COLUMN) expression to have type bytes"); errors.add("value type string doesn't match type bytes of column"); errors.add("as decimal, found type: int"); errors.add("to be of type decimal, found type float"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index c32fa76eb..c8bfa3c6a 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -38,6 +38,7 @@ public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { @Override public void buildStatement() { errors.add("and thus is not indexable"); // array types are not indexable + errors.add("context-dependent operators are not allowed in STORED COMPUTED COLUMN"); if (globalState.getDbmsSpecificOptions().testTempTables) { errors.add("constraints on temporary tables may reference only temporary tables"); errors.add("constraints on permanent tables may reference only permanent tables"); From 18b35964733949593c2d6ffb771b245a5ee7d2aa Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Thu, 27 Apr 2023 19:32:01 +0200 Subject: [PATCH 340/856] Materialize: Typo in error message (#782) Caused by: org.postgresql.util.PSQLException: ERROR: result exceeds max size of 1073.7 MB --- src/sqlancer/materialize/gen/MaterializeCommon.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/gen/MaterializeCommon.java b/src/sqlancer/materialize/gen/MaterializeCommon.java index 7d722e102..7dee348f4 100644 --- a/src/sqlancer/materialize/gen/MaterializeCommon.java +++ b/src/sqlancer/materialize/gen/MaterializeCommon.java @@ -31,7 +31,7 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("non-integer constant in GROUP BY"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY position"); - errors.add("results exceeds max size of"); + errors.add("result exceeds max size of"); errors.add("does not exist"); errors.add("aggregate functions are not allowed in"); From 9e17ede91412199f2633d8b1c724e1ee37f76de1 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Sat, 20 May 2023 21:43:51 +0800 Subject: [PATCH 341/856] fix: partly fix ci error in databend (#784) * fix: fix ci error https://github.com/sqlancer/sqlancer/actions/runs/4822718555/jobs/8590290712 * style: run mvn format * fix: fix bool type size * fix: Nullable type * fix: add FLOAT64 * refactor: change the sequence correspond to https://databend.rs/doc/sql-reference/data-types/#general-data-types --- src/sqlancer/databend/DatabendSchema.java | 47 +++++++++++++++-------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 2e8e3c2bf..1400d9d5e 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -232,48 +232,65 @@ public DatabendTables getRandomTableNonEmptyAndViewTables() { } private static DatabendCompositeDataType getColumnType(String typeString) { - DatabendDataType primitiveType; - int size = -1; if (typeString.startsWith("DECIMAL")) { // Ugly hack return new DatabendCompositeDataType(DatabendDataType.FLOAT, 8); } - switch (typeString) { - case "INT": + if (typeString.startsWith("Nullable")) { // Ugly hack + String substring = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + return getColumnTypeNormalCases(substring); + } + return getColumnTypeNormalCases(typeString); + } + + private static DatabendCompositeDataType getColumnTypeNormalCases(String typeString) { + DatabendDataType primitiveType; + int size = -1; + switch (typeString.toUpperCase()) { + case "BOOLEAN": + case "BOOL": + primitiveType = DatabendDataType.BOOLEAN; + size = 1; + break; + case "TINYINT": + case "INT8": primitiveType = INT; - size = 4; + size = 1; break; case "SMALLINT": + case "INT16": primitiveType = INT; size = 2; break; - case "BIGINT": + case "INT": + case "INT32": primitiveType = INT; - size = 8; + size = 4; break; - case "TINYINT": + case "BIGINT": + case "INT64": primitiveType = INT; - size = 1; - break; - case "VARCHAR": - primitiveType = DatabendDataType.VARCHAR; + size = 8; break; case "FLOAT": + case "FLOAT32": primitiveType = DatabendDataType.FLOAT; size = 4; break; case "DOUBLE": + case "FLOAT64": primitiveType = DatabendDataType.FLOAT; size = 8; break; - case "BOOLEAN": - primitiveType = DatabendDataType.BOOLEAN; - break; // case "DATE": // primitiveType = DatabendDataType.DATE; // break; // case "TIMESTAMP": // primitiveType = DatabendDataType.TIMESTAMP; // break; + case "VARCHAR": + case "STRING": + primitiveType = DatabendDataType.VARCHAR; + break; case "NULL": primitiveType = DatabendDataType.NULL; break; From e696c6e93c6f88e9edf173b482cff65ff82184c8 Mon Sep 17 00:00:00 2001 From: huang xu Date: Mon, 22 May 2023 11:29:53 +0800 Subject: [PATCH 342/856] Support Apache Doris (#758) * support doris init commit * support doris complete Signed-off-by: ChaseHuangxu * code format & correct github workflow main.yml about doris & add doris to README.md support dbms * correct README.md support dbms * change sql union method to union all in TLPDistinct check Signed-off-by: ChaseHuangxu * Limit the data format inserted by decimal type, and support Doris function call Signed-off-by: ChaseHuangxu * correct github/workflows/DBMS Tests(doris) Signed-off-by: ChaseHuangxu --------- Signed-off-by: ChaseHuangxu --- .github/workflows/main.yml | 43 +- README.md | 3 +- src/check_names.py | 1 + src/sqlancer/doris/DorisBugs.java | 45 ++ src/sqlancer/doris/DorisErrors.java | 64 ++ src/sqlancer/doris/DorisOptions.java | 194 +++++ src/sqlancer/doris/DorisProvider.java | 155 ++++ src/sqlancer/doris/DorisSchema.java | 611 ++++++++++++++++ .../doris/ast/DorisAggregateOperation.java | 39 + .../doris/ast/DorisBetweenOperation.java | 39 + .../ast/DorisBinaryArithmeticOperation.java | 143 ++++ .../ast/DorisBinaryComparisonOperation.java | 120 +++ .../ast/DorisBinaryLogicalOperation.java | 130 ++++ .../doris/ast/DorisCaseOperation.java | 38 + .../doris/ast/DorisCastOperation.java | 46 ++ src/sqlancer/doris/ast/DorisColumnValue.java | 53 ++ src/sqlancer/doris/ast/DorisConstant.java | 681 ++++++++++++++++++ src/sqlancer/doris/ast/DorisExpression.java | 13 + .../doris/ast/DorisFunctionOperation.java | 282 ++++++++ src/sqlancer/doris/ast/DorisInOperation.java | 53 ++ src/sqlancer/doris/ast/DorisJoin.java | 108 +++ .../doris/ast/DorisLikeOperation.java | 85 +++ src/sqlancer/doris/ast/DorisOrderByTerm.java | 10 + src/sqlancer/doris/ast/DorisSelect.java | 41 ++ .../doris/ast/DorisUnaryPostfixOperation.java | 87 +++ .../doris/ast/DorisUnaryPrefixOperation.java | 112 +++ .../doris/gen/DorisAlterTableGenerator.java | 50 ++ .../doris/gen/DorisDeleteGenerator.java | 31 + .../doris/gen/DorisDropTableGenerator.java | 28 + .../doris/gen/DorisDropViewGenerator.java | 27 + .../doris/gen/DorisIndexGenerator.java | 46 ++ .../doris/gen/DorisInsertGenerator.java | 56 ++ .../gen/DorisNewExpressionGenerator.java | 427 +++++++++++ .../gen/DorisRandomQuerySynthesizer.java | 79 ++ .../doris/gen/DorisTableGenerator.java | 114 +++ .../doris/gen/DorisUpdateGenerator.java | 56 ++ .../doris/gen/DorisViewGenerator.java | 38 + .../doris/oracle/DorisNoRECOracle.java | 137 ++++ .../DorisPivotedQuerySynthesisOracle.java | 155 ++++ ...DorisQueryPartitioningAggregateTester.java | 194 +++++ .../tlp/DorisQueryPartitioningBase.java | 98 +++ .../DorisQueryPartitioningDistinctTester.java | 45 ++ .../DorisQueryPartitioningGroupByTester.java | 55 ++ .../DorisQueryPartitioningHavingTester.java | 73 ++ .../DorisQueryPartitioningWhereTester.java | 53 ++ .../doris/utils/DorisNumberUtils.java | 111 +++ .../visitor/DorisExpectedValueVisitor.java | 155 ++++ .../doris/visitor/DorisExprToNode.java | 25 + .../doris/visitor/DorisToStringVisitor.java | 175 +++++ test/sqlancer/dbms/TestDoris.java | 54 ++ 50 files changed, 5476 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/doris/DorisBugs.java create mode 100644 src/sqlancer/doris/DorisErrors.java create mode 100644 src/sqlancer/doris/DorisOptions.java create mode 100644 src/sqlancer/doris/DorisProvider.java create mode 100644 src/sqlancer/doris/DorisSchema.java create mode 100644 src/sqlancer/doris/ast/DorisAggregateOperation.java create mode 100644 src/sqlancer/doris/ast/DorisBetweenOperation.java create mode 100644 src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java create mode 100644 src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java create mode 100644 src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java create mode 100644 src/sqlancer/doris/ast/DorisCaseOperation.java create mode 100644 src/sqlancer/doris/ast/DorisCastOperation.java create mode 100644 src/sqlancer/doris/ast/DorisColumnValue.java create mode 100644 src/sqlancer/doris/ast/DorisConstant.java create mode 100644 src/sqlancer/doris/ast/DorisExpression.java create mode 100644 src/sqlancer/doris/ast/DorisFunctionOperation.java create mode 100644 src/sqlancer/doris/ast/DorisInOperation.java create mode 100644 src/sqlancer/doris/ast/DorisJoin.java create mode 100644 src/sqlancer/doris/ast/DorisLikeOperation.java create mode 100644 src/sqlancer/doris/ast/DorisOrderByTerm.java create mode 100644 src/sqlancer/doris/ast/DorisSelect.java create mode 100644 src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java create mode 100644 src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java create mode 100644 src/sqlancer/doris/gen/DorisAlterTableGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisDeleteGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisDropTableGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisDropViewGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisIndexGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisInsertGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisNewExpressionGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java create mode 100644 src/sqlancer/doris/gen/DorisTableGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisUpdateGenerator.java create mode 100644 src/sqlancer/doris/gen/DorisViewGenerator.java create mode 100644 src/sqlancer/doris/oracle/DorisNoRECOracle.java create mode 100644 src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java create mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/doris/utils/DorisNumberUtils.java create mode 100644 src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java create mode 100644 src/sqlancer/doris/visitor/DorisExprToNode.java create mode 100644 src/sqlancer/doris/visitor/DorisToStringVisitor.java create mode 100644 test/sqlancer/dbms/TestDoris.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d4166e941..c03e3908c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -442,11 +442,52 @@ jobs: - name: Setup Yugabyte run: | docker pull yugabytedb/yugabyte:latest - docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false + docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false sleep 5 - name: Run Tests run: YUGABYTE_AVAILABLE=true mvn -Dtest=TestYugabyte test + + doris: + name: DBMS Tests (Apache Doris) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1 + with: + java-version: 11 + - name: install mysql client + run: | + sudo apt update + sudo apt install mysql-client --assume-yes + - name: Set up Apache Doris + run: | + sudo sysctl -w vm.max_map_count=2000000 + LASTEST_TAG=$(curl -s GET https://api.github.com/repos/apache/doris/releases | jq -r '.[].tag_name' | sed -n 1p) + LASTEST_TAG_BIG_VERSION=$(echo ${LASTEST_TAG} | awk '{split($1, arr, "."); print arr[1]"."arr[2]}') + curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz" + curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz" + curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz" + mkdir ./doris + tar xf apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-fe-${LASTEST_TAG}-bin-x86_64 doris/fe + tar xf apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-be-${LASTEST_TAG}-bin-x86_64 doris/be + tar xf apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64 doris/dependencies + cp doris/dependencies/*.jar doris/be/lib/ + doris/fe/bin/start_fe.sh --daemon + doris/be/bin/start_be.sh --daemon + sudo apt install libnet-ifconfig-wrapper-perl --assume-yes + IP=$(ifconfig eth0 | grep inet | grep -v inet6 | awk '{print $2}') + mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';" + mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: DORIS_AVAILABLE=true mvn -Dtest=TestDoris test + + java13: name: Java 13 Compatibility (DuckDB) runs-on: ubuntu-latest diff --git a/README.md b/README.md index 59f69cce2..f22b00f4d 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,8 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | Databend | Working | Typed | | | QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | | CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | -| Materialize |Working | Typed | | +| Materialize |Working | Typed | +| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. # Using SQLancer diff --git a/src/check_names.py b/src/check_names.py index 6dd4f1a72..1b110370b 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -52,4 +52,5 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["SQLite3"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "sqlite3")) name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes + name_to_files["Doris"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "doris")) verify_all_dbs(name_to_files) diff --git a/src/sqlancer/doris/DorisBugs.java b/src/sqlancer/doris/DorisBugs.java new file mode 100644 index 000000000..080aa0450 --- /dev/null +++ b/src/sqlancer/doris/DorisBugs.java @@ -0,0 +1,45 @@ +package sqlancer.doris; + +public final class DorisBugs { + // https://github.com/apache/doris/issues/17697 + // Logical bug about where true not in (columns) + public static boolean bug17697 = true; + + // https://github.com/apache/doris/issues/17700 + // Cannot use between and in boolean column + public static boolean bug17700 = true; + + // https://github.com/apache/doris/issues/17701 + // Wrong result of `where column not in (values)` + public static boolean bug17701 = true; + + // https://github.com/apache/doris/issues/17705 + // Different result caused by `where` split and union all + public static boolean bug17705 = true; + + // https://github.com/apache/doris/issues/19370 + // Internal Error occur in GroupBy&Having sql + // fixed by https://github.com/apache/doris/pull/19559 + public static boolean bug19370 = true; + + // https://github.com/apache/doris/issues/19374 + // Different result of having not ($value in column) and having ($value not in column) + // fixed by https://github.com/apache/doris/pull/19471 + public static boolean bug19374 = true; + + // https://github.com/apache/doris/issues/19611 + // ERROR occur in nested subqueries with same column name and union + public static boolean bug19611 = true; + + // https://github.com/apache/doris/issues/19613 + // Wrong result when right outer join and where false + public static boolean bug19613 = true; + + // https://github.com/apache/doris/issues/19614 + // Wrong result when value like column from table_join + public static boolean bug19614 = true; + + private DorisBugs() { + + } +} diff --git a/src/sqlancer/doris/DorisErrors.java b/src/sqlancer/doris/DorisErrors.java new file mode 100644 index 000000000..678601635 --- /dev/null +++ b/src/sqlancer/doris/DorisErrors.java @@ -0,0 +1,64 @@ +package sqlancer.doris; + +import sqlancer.common.query.ExpectedErrors; + +public final class DorisErrors { + + private DorisErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + // SQL syntax error + errors.add("Syntax error"); + errors.add("Please check your sql, we meet an error when parsing"); + errors.add("but returns type"); + errors.add("is not a number"); + + // Not in line with Doris' logic + errors.add("Unexpected exception: null"); + errors.add("Cross join can't be used with ON clause"); + errors.add("BetweenPredicate needs to be rewritten into a CompoundPredicate"); + errors.add("can't be assigned to some PlanNode"); + errors.add("can not cast from origin type"); + errors.add("not produced by aggregation output"); + errors.add("cannot combine"); // cannot combine SELECT DISTINCT with aggregate functions or GROUP BY + errors.add("Invalid type"); + errors.add("cannot be cast to"); + + // functions + errors.add("No matching function with signature"); + errors.add("Invalid number format"); + errors.add("group_concat requires"); + errors.add("function's argument should be"); + errors.add("requires a numeric parameter"); + errors.add("out of bounds"); + errors.add("function do not support"); + errors.add("parameter must be"); + errors.add("Not supported input arguments types"); + errors.add("No matching function with signature"); + errors.add("function"); + errors.add("Invalid"); + errors.add("Incorrect"); + + // regex + + // To avoid bugs + if (DorisBugs.bug19370) { + errors.add("failed to initialize storage"); + } + if (DorisBugs.bug19374) { + errors.add("the size of the result sets mismatch"); + } + if (DorisBugs.bug19611) { + errors.add("Duplicated inline view column alias"); + } + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.add("Insert has filtered data in strict mode"); + errors.add("Only value columns of unique table could be updated"); + errors.add("Only unique olap table could be updated"); + errors.add("Number out of range"); + } + +} diff --git a/src/sqlancer/doris/DorisOptions.java b/src/sqlancer/doris/DorisOptions.java new file mode 100644 index 000000000..a2a05b95e --- /dev/null +++ b/src/sqlancer/doris/DorisOptions.java @@ -0,0 +1,194 @@ +package sqlancer.doris; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.doris.DorisOptions.DorisOracleFactory; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.oracle.DorisNoRECOracle; +import sqlancer.doris.oracle.DorisPivotedQuerySynthesisOracle; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningAggregateTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningDistinctTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningGroupByTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningHavingTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningWhereTester; + +@Parameters(commandDescription = "Apache Doris (default port: " + DorisOptions.DEFAULT_PORT + ", default host: " + + DorisOptions.DEFAULT_HOST + ")") +public class DorisOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 9030; + + @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables/views that can be created") + public int maxNumTables = 10; + + @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") + public int maxNumIndexes = 20; + + @Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1) + public boolean testDefaultValues = true; + + @Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1) + public boolean testNotNullConstraints = true; + + @Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1) + public boolean testFunctions; + + @Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1) + public boolean testCasts = true; + + @Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions", arity = 1) + public boolean testBetween = true; + + @Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1) + public boolean testIn = true; + + @Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1) + public boolean testCase = true; + + @Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1) + public boolean testBinaryLogicals = true; + + @Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1) + public boolean testIntConstants = true; + + @Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1) + public boolean testFloatConstants = true; + + @Parameter(names = "--test-decimal-constants", description = "Allow generating DECIMAL constants", arity = 1) + public boolean testDecimalConstants = true; + + @Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testDateConstants = true; + + @Parameter(names = "--test-datetime-constants", description = "Allow generating DATETIME constants", arity = 1) + public boolean testDateTimeConstants = true; + + @Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1) + public boolean testStringConstants = true; + + @Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1) + public boolean testBooleanConstants = true; + + @Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1) + public boolean testBinaryComparisons = true; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates; + + @Parameter(names = "--max-num-table-alters", description = "The maximum number of ALTER TABLE statements that are issued for a database", arity = 1) + public int maxNumTableAlters; + + @Parameter(names = "--test-engine-type", description = "The engine type in Doris, only consider OLAP now", arity = 1) + public String testEngineType = "OLAP"; // skip now + + @Parameter(names = "--test-indexes", description = "Allow explicit indexes, Doris only supports creating indexes on single-column BITMAP", arity = 1) + public boolean testIndexes = true; // skip now + + @Parameter(names = "--test-column-aggr", description = "Allow test column aggregation (sum, min, max, replace, replace_if_not_null, hll_union, bitmap_untion)", arity = 1) + public boolean testColumnAggr = true; + + @Parameter(names = "--test-datemodel", description = "Allow generating Doris’s data model in tables. (Aggregate、Uniqe、Duplicate)", arity = 1) + public boolean testDataModel = true; + + @Parameter(names = "--test-distribution", description = "Allow generating data distribution in tables.", arity = 1) + public boolean testDistribution = true; // must have it, skip now + + @Parameter(names = "--test-rollup", description = "Allow generating rollups in tables.", arity = 1) + public boolean testRollup = true; // skip now + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(DorisOracleFactory.NOREC); + + public enum DorisOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisNoRECOracle(globalState); + } + + }, + HAVING { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + return new DorisQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DorisGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); + oracles.add(new DorisQueryPartitioningWhereTester(globalState)); + oracles.add(new DorisQueryPartitioningHavingTester(globalState)); + oracles.add(new DorisQueryPartitioningAggregateTester(globalState)); + oracles.add(new DorisQueryPartitioningDistinctTester(globalState)); + oracles.add(new DorisQueryPartitioningGroupByTester(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + PQS { + @Override + public TestOracle create(DorisGlobalState globalState) throws Exception { + return new DorisPivotedQuerySynthesisOracle(globalState); + } + }, + ALL { + @Override + public TestOracle create(DorisGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(new DorisNoRECOracle(globalState)); + oracles.add(new DorisQueryPartitioningWhereTester(globalState)); + oracles.add(new DorisQueryPartitioningHavingTester(globalState)); + oracles.add(new DorisQueryPartitioningAggregateTester(globalState)); + oracles.add(new DorisQueryPartitioningDistinctTester(globalState)); + oracles.add(new DorisQueryPartitioningGroupByTester(globalState)); + oracles.add(new DorisPivotedQuerySynthesisOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + + } + + @Override + public List getTestOracleFactory() { + return oracles; + } + +} diff --git a/src/sqlancer/doris/DorisProvider.java b/src/sqlancer/doris/DorisProvider.java new file mode 100644 index 000000000..3f64231f1 --- /dev/null +++ b/src/sqlancer/doris/DorisProvider.java @@ -0,0 +1,155 @@ +package sqlancer.doris; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.gen.DorisAlterTableGenerator; +import sqlancer.doris.gen.DorisDeleteGenerator; +import sqlancer.doris.gen.DorisDropTableGenerator; +import sqlancer.doris.gen.DorisDropViewGenerator; +import sqlancer.doris.gen.DorisIndexGenerator; +import sqlancer.doris.gen.DorisInsertGenerator; +import sqlancer.doris.gen.DorisTableGenerator; +import sqlancer.doris.gen.DorisUpdateGenerator; +import sqlancer.doris.gen.DorisViewGenerator; + +@AutoService(DatabaseProvider.class) +public class DorisProvider extends SQLProviderAdapter { + + public DorisProvider() { + super(DorisGlobalState.class, DorisOptions.class); + } + + public enum Action implements AbstractAction { + CREATE_TABLE(DorisTableGenerator::createRandomTableStatement), CREATE_VIEW(DorisViewGenerator::getQuery), + CREATE_INDEX(DorisIndexGenerator::getQuery), INSERT(DorisInsertGenerator::getQuery), + DELETE(DorisDeleteGenerator::generate), UPDATE(DorisUpdateGenerator::getQuery), + ALTER_TABLE(DorisAlterTableGenerator::getQuery), + TRUNCATE((g) -> new SQLQueryAdapter( + "TRUNCATE TABLE " + g.getSchema().getRandomTable(t -> !t.isView()).getName())), + DROP_TABLE(DorisDropTableGenerator::dropTable), DROP_VIEW(DorisDropViewGenerator::dropView); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(DorisGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(DorisGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case DELETE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes); + case UPDATE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates); + case ALTER_TABLE: + return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumTableAlters); + case TRUNCATE: + return r.getInteger(0, 2); + case CREATE_TABLE: + case CREATE_INDEX: + case CREATE_VIEW: + case DROP_TABLE: + case DROP_VIEW: + return 0; + default: + throw new AssertionError(a); + } + } + + public static class DorisGlobalState extends SQLGlobalState { + + @Override + protected DorisSchema readSchema() throws SQLException { + return DorisSchema.fromConnection(getConnection(), getDatabaseName()); + } + + } + + @Override + public void generateDatabase(DorisGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success = false; + do { + SQLQueryAdapter qt = new DorisTableGenerator().getQuery(globalState); + if (qt != null) { + success = globalState.executeStatement(qt); + } + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + DorisProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(DorisGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + if (password.equals("\"\"")) { + password = ""; + } + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = DorisOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = DorisOptions.DEFAULT_PORT; + } + String databaseName = globalState.getDatabaseName(); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "doris"; + } + +} diff --git a/src/sqlancer/doris/DorisSchema.java b/src/sqlancer/doris/DorisSchema.java new file mode 100644 index 000000000..d452f784a --- /dev/null +++ b/src/sqlancer/doris/DorisSchema.java @@ -0,0 +1,611 @@ +package sqlancer.doris; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractRowValue; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.ast.DorisConstant; + +public class DorisSchema extends AbstractSchema { + + public enum DorisTableDataModel { + UNIQUE, AGGREGATE, DUPLICATE; + + public static DorisTableDataModel getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum DorisColumnAggrType { + SUM, MIN, MAX, REPLACE, REPLCAE_IF_NOT_NULL, BITMAP_UNION, HLL_UNION, NULL; + + public static DorisColumnAggrType getRandom(DorisCompositeDataType columnDataType) { + // if (columnDataType.getPrimitiveDataType() == DorisSchema.DorisDataType.BITMAP) { + // return DorisColumnAggrType.BITMAP_UNION; + // } + // if (columnDataType.getPrimitiveDataType() == DorisSchema.DorisDataType.HLL) { + // return DorisColumnAggrType.HLL_UNION; + // } + + return Randomly.fromOptions(SUM, MIN, MAX, REPLACE, REPLCAE_IF_NOT_NULL); + } + } + + public enum DorisDataType { + INT, FLOAT, DECIMAL, DATE, DATETIME, VARCHAR, BOOLEAN, NULL; + // HLL, BITMAP, ARRAY; + + private int decimalScale; + private int decimalPrecision; + private int varcharLength; + + public static DorisDataType getRandomWithoutNull() { + DorisDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DorisDataType.NULL); + return dt; + } + + public int getDecimalScale() { + return decimalScale; + } + + public void setDecimalScale(int decimalScale) { + this.decimalScale = decimalScale; + } + + public int getDecimalPrecision() { + return decimalPrecision; + } + + public void setDecimalPrecision(int decimalPrecision) { + this.decimalPrecision = decimalPrecision; + } + + public int getVarcharLength() { + return varcharLength; + } + + public void setVarcharLength(int varcharLength) { + this.varcharLength = varcharLength; + } + } + + public static class DorisCompositeDataType { + + private final DorisDataType dataType; + + private final int size; + + public DorisCompositeDataType(DorisDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public DorisDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static DorisCompositeDataType getRandomWithoutNull() { + DorisDataType type = DorisDataType.getRandomWithoutNull(); + int size = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8, 16); + break; + case FLOAT: + size = Randomly.fromOptions(4, 12); + break; + case DECIMAL: + size = Randomly.fromOptions(1, 3); // DECIMAL or DECIMALV3 + break; + case DATE: + case DATETIME: + case VARCHAR: + case BOOLEAN: + // case HLL: + // case BITMAP: + // case ARRAY: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new DorisCompositeDataType(type, size); + } + + public void initColumnArgs() { + Randomly r = new Randomly(); + int scale; + int precision; + int varcharLength; + switch (getPrimitiveDataType()) { + case DECIMAL: + if (getPrimitiveDataType().getDecimalPrecision() != 0) { + break; + } + if (size == 1) { + scale = r.getInteger(0, 9); + precision = r.getInteger(scale + 1, scale + 18); + getPrimitiveDataType().setDecimalPrecision(precision); + getPrimitiveDataType().setDecimalScale(scale); + } else if (size == 3) { + precision = r.getInteger(1, 38); + scale = r.getInteger(0, precision); + getPrimitiveDataType().setDecimalPrecision(precision); + getPrimitiveDataType().setDecimalScale(scale); + } else { + throw new AssertionError(size); + } + break; + case VARCHAR: + if (getPrimitiveDataType().getVarcharLength() != 0) { + break; + } + varcharLength = r.getInteger(1, 255); + getPrimitiveDataType().setVarcharLength(varcharLength); + break; + default: + // pass + } + + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 16: + return "LARGEINT"; + case 8: + return "BIGINT"; + case 4: + return "INT"; + case 2: + return "SMALLINT"; + case 1: + return "TINYINT"; + default: + throw new AssertionError(size); + } + case FLOAT: + switch (size) { + case 12: + return "DOUBLE"; + case 4: + return "FLOAT"; + default: + throw new AssertionError(size); + } + case DECIMAL: + switch (size) { + case 1: + return "DECIMAL(" + getPrimitiveDataType().getDecimalPrecision() + "," + + getPrimitiveDataType().getDecimalScale() + ")"; + case 3: + return "DECIMALV3(" + getPrimitiveDataType().getDecimalPrecision() + "," + + getPrimitiveDataType().getDecimalScale() + ")"; + default: + throw new AssertionError(size); + } + case DATE: + return "DATEV2"; + case DATETIME: + return Randomly.fromOptions("DATETIME", "DATETIMEV2"); + case VARCHAR: + return Randomly.fromOptions("VARCHAR", "CHAR") + "(" + getPrimitiveDataType().getVarcharLength() + ")"; + case BOOLEAN: + return "BOOLEAN"; + // case HLL: + // return "HLL"; + // case BITMAP: + // return "BITMAP"; + // case ARRAY: + // return "ARRAY"; + case NULL: + return Randomly.fromOptions("NULL"); + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + public boolean canBeKey() { + switch (dataType) { + // case HLL: + // case BITMAP: + // case ARRAY: + case FLOAT: + return false; + default: + return true; + } + } + + } + + public static class DorisColumn extends AbstractTableColumn { + + private final boolean isKey; + private final boolean isNullable; + private final DorisColumnAggrType aggrType; + private final boolean hasDefaultValue; + private final String defaultValue; + + public DorisColumn(String name, DorisCompositeDataType type, boolean isKey, boolean isNullable, + DorisColumnAggrType aggrType, boolean hasDefaultValue, String defaultValue) { + super(name, null, type); + this.isKey = isKey; + this.isNullable = isNullable; + this.aggrType = aggrType; + this.hasDefaultValue = hasDefaultValue; + this.defaultValue = defaultValue; + } + + public DorisColumn(String name, DorisCompositeDataType type, boolean isKey, boolean isNullable) { + super(name, null, type); + this.isKey = isKey; + this.isNullable = isNullable; + this.aggrType = DorisColumnAggrType.NULL; + this.hasDefaultValue = false; + this.defaultValue = ""; + } + + public boolean isKey() { + return isKey; + } + + public boolean isNullable() { + return isNullable; + } + + public boolean hasDefaultValue() { + return hasDefaultValue; + } + + @Override + public String toString() { + String ret = this.getName() + " " + this.getType(); + if (aggrType != DorisColumnAggrType.NULL) { + ret += " " + aggrType.name(); + } + if (!isNullable) { + ret += " NOT NULL"; + } + if (hasDefaultValue) { + ret += " DEFAULT " + defaultValue; + } + return ret; + } + + @Override + public int compareTo(AbstractTableColumn o) { + // To sort columns + DorisColumn other = (DorisColumn) o; + if (isKey != other.isKey) { + return isKey ? 1 : -1; + } + return getName().compareTo(other.getName()); + } + } + + public static class DorisTables extends AbstractTables { + + public DorisTables(List tables) { + super(tables); + } + + public DorisRowValue getRandomRowValue(SQLConnection con) throws SQLException { + String rowValueQuery = String.format("SELECT %s FROM %s ORDER BY 1 LIMIT 1", columnNamesAsString( + c -> c.getTable().getName() + "." + c.getName() + " AS " + c.getTable().getName() + c.getName()), + tableNamesAsString()); + Map values = new HashMap<>(); + try (Statement s = con.createStatement()) { + ResultSet rs = s.executeQuery(rowValueQuery); + if (!rs.next()) { + throw new IgnoreMeException(); + // throw new AssertionError("could not find random row " + rowValueQuery + "\n"); + } + for (int i = 0; i < getColumns().size(); i++) { + DorisColumn column = getColumns().get(i); + int columnIndex = rs.findColumn(column.getTable().getName() + column.getName()); + assert columnIndex == i + 1; + DorisConstant constant; + if (rs.getString(columnIndex) == null) { + constant = DorisConstant.createNullConstant(); + } else { + switch (column.getType().getPrimitiveDataType()) { + case INT: + constant = DorisConstant.createIntConstant(rs.getLong(columnIndex)); + break; + case FLOAT: + case DECIMAL: + constant = DorisConstant.createFloatConstant(rs.getDouble(columnIndex)); + break; + case DATE: + constant = DorisConstant.createDateConstant(rs.getString(columnIndex)); + break; + case DATETIME: + constant = DorisConstant.createDatetimeConstant(rs.getString(columnIndex)); + break; + case VARCHAR: + constant = DorisConstant.createStringConstant(rs.getString(columnIndex)); + break; + case BOOLEAN: + constant = DorisConstant.createBooleanConstant(rs.getBoolean(columnIndex)); + break; + case NULL: + constant = DorisConstant.createNullConstant(); + break; + default: + throw new IgnoreMeException(); + } + } + values.put(column, constant); + } + assert !rs.next(); + return new DorisRowValue(this, values); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } + + } + + public static class DorisRowValue extends AbstractRowValue { + + DorisRowValue(DorisTables tables, Map values) { + super(tables, values); + } + + } + + public DorisSchema(List databaseTables) { + super(databaseTables); + } + + public DorisTables getRandomTableNonEmptyTables() { + return new DorisTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public DorisTables getRandomTableNonEmptyAndViewTables() { + List tables = getDatabaseTables().stream().filter(t -> !t.isView()).collect(Collectors.toList()); + tables = Randomly.nonEmptySubset(tables); + return new DorisTables(tables); + } + + public int getIndexCount() { + int count = 0; + for (DorisTable table : getDatabaseTables()) { + count += table.getIndexes().size(); + } + return count; + } + + private static DorisCompositeDataType getColumnType(String typeString) { + DorisDataType primitiveType; + int size = -1; + + if (typeString.startsWith("DECIMALV3")) { + primitiveType = DorisDataType.DECIMAL; + String precisionAndScale = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + String[] split = precisionAndScale.split(","); + assert split.length == 2; + primitiveType.setDecimalPrecision(Integer.parseInt(split[0].trim())); + primitiveType.setDecimalScale(Integer.parseInt(split[1].trim())); + size = 3; + } else if (typeString.startsWith("DECIMAL")) { + primitiveType = DorisDataType.DECIMAL; + String precisionAndScale = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + String[] split = precisionAndScale.split(","); + assert split.length == 2; + primitiveType.setDecimalPrecision(Integer.parseInt(split[0].trim())); + primitiveType.setDecimalScale(Integer.parseInt(split[1].trim())); + size = 1; + } else if (typeString.startsWith("DATEV2")) { + primitiveType = DorisDataType.DATE; + size = 2; + } else if (typeString.startsWith("DATE")) { + primitiveType = DorisDataType.DATE; + size = 1; + } else if (typeString.startsWith("DATETIMEV2")) { + primitiveType = DorisDataType.DATETIME; + size = 2; + } else if (typeString.startsWith("DATETIME")) { + primitiveType = DorisDataType.DATETIME; + size = 1; + } else if (typeString.startsWith("CHAR") || typeString.startsWith("VARCHAR")) { + primitiveType = DorisDataType.VARCHAR; + String varcharLength = typeString.substring(typeString.indexOf('(') + 1, typeString.indexOf(')')); + primitiveType.setVarcharLength(Integer.parseInt(varcharLength.trim())); + } else { + switch (typeString) { + case "LARGEINT": + primitiveType = DorisDataType.INT; + size = 16; + break; + case "BIGINT": + primitiveType = DorisDataType.INT; + size = 8; + break; + case "INT": + primitiveType = DorisDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = DorisDataType.INT; + size = 2; + break; + case "TINYINT": + primitiveType = DorisDataType.INT; + size = 1; + break; + case "DOUBLE": + primitiveType = DorisDataType.FLOAT; + size = 12; + break; + case "FLOAT": + primitiveType = DorisDataType.FLOAT; + size = 4; + break; + case "DECIMAL": + case "DECIMAL(*,*)": + primitiveType = DorisDataType.DECIMAL; + size = 1; + break; + case "DECIMALV3": + case "DECIMALV3(*,*)": + primitiveType = DorisDataType.DECIMAL; + size = 3; + break; + case "CHAR": + case "CHAR(*)": + case "VARCHAR": + case "VARCHAR(*)": + primitiveType = DorisDataType.VARCHAR; + break; + case "DATE": + primitiveType = DorisDataType.DATE; + size = 1; + break; + case "DATEV2": + primitiveType = DorisDataType.DATE; + size = 2; + break; + case "DATETIME": + primitiveType = DorisDataType.DATETIME; + size = 1; + break; + case "DATETIMEV2": + primitiveType = DorisDataType.DATETIME; + size = 2; + break; + case "BOOLEAN": + primitiveType = DorisDataType.BOOLEAN; + break; + // case "HLL": + // primitiveType = DorisDataType.HLL; + // break; + // case "BITMAP": + // primitiveType = DorisDataType.BITMAP; + // break; + case "NULL": + primitiveType = DorisDataType.NULL; + break; + default: + throw new AssertionError(typeString); + } + } + return new DorisCompositeDataType(primitiveType, size); + } + + public static class DorisTable extends AbstractRelationalTable { + + public DorisTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + public List getRandomNonEmptyInsertColumns() { + List columns = getColumns(); + List retColumns = new ArrayList<>(); + List remainColumns = new ArrayList<>(); + for (DorisColumn column : columns) { + if (!column.hasDefaultValue() && !column.isNullable) { + retColumns.add(column); + } else { + remainColumns.add(column); + } + } + if (retColumns.isEmpty()) { + retColumns.addAll(Randomly.nonEmptySubset(remainColumns)); + } else { + retColumns.addAll(Randomly.subset(remainColumns)); + } + return retColumns; + } + + } + + public static DorisSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + DorisTable t = new DorisTable(tableName, databaseColumns, isView); + for (DorisColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new DorisSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES")) { + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("DESCRIBE " + tableName)) { + while (rs.next()) { + String columnName = rs.getString("Field"); + String dataType = rs.getString("Type"); + String isNullString = rs.getString("Null"); + assert isNullString.contentEquals("Yes") || isNullString.contentEquals("No"); + boolean isNullable = isNullString.contentEquals("Yes"); + String isKeyString = rs.getString("Key"); + assert isKeyString.contentEquals("true") || isKeyString.contentEquals("false"); + boolean isKey = isKeyString.contentEquals("true"); + String defaultValue = rs.getString("Default"); + boolean hasDefaultValue = defaultValue != null; + DorisColumn c = new DorisColumn(columnName, getColumnType(dataType), isKey, isNullable, + DorisColumnAggrType.NULL, hasDefaultValue, defaultValue); + columns.add(c); + } + } + } + return columns; + } + +} diff --git a/src/sqlancer/doris/ast/DorisAggregateOperation.java b/src/sqlancer/doris/ast/DorisAggregateOperation.java new file mode 100644 index 000000000..de818c26d --- /dev/null +++ b/src/sqlancer/doris/ast/DorisAggregateOperation.java @@ -0,0 +1,39 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisAggregateOperation extends + NewFunctionNode implements DorisExpression { + + public DorisAggregateOperation(List args, DorisAggregateFunction func) { + super(DorisExprToNode.casts(args), func); + } + + public enum DorisAggregateFunction { + COLLECT_SET(1), MIN(1), STDDEV_SAMP(1), AVG(1), AVG_WEIGHTED(2), PERCENTILE(1), PERCENTILE_ARRAY(2), + HLL_UNION_AGG(1), TOPN(2), TOPN_ARRAY(2), TOPN_WEIGHTED(3), COUNT(1), SUM(1), MAX_BY(2), BITMAP_UNION(1), + GROUP_BITMAP_XOR(1), GROUP_BIT_ADD(1), GROUP_BIT_OR(1), GROUP_BIT_XOR(1), PERCENTILE_APPROX(2), STDDEV(1), + STDDEV_POP(1), GROUP_CONCAT(1), COLLECT_LIST(1), MIN_BY(2), MAX(1), ANY_VALUE(1), VAR_SAMP(1), VARIANCE_SAMP(1), + APPROX_COUNT_DISTINCT(1), VARIANCE(1), VAR_POP(1), VARIANCE_POP(1), GROUPING(1), GROUPING_ID(1); + // RETENTION(1), SEQUENCE_MATCH(1), SEQUENCE_COUNT(1), // TODO，not currently considered + + private int nrArgs; + + DorisAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static DorisAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } +} diff --git a/src/sqlancer/doris/ast/DorisBetweenOperation.java b/src/sqlancer/doris/ast/DorisBetweenOperation.java new file mode 100644 index 000000000..76488393b --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBetweenOperation.java @@ -0,0 +1,39 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisBetweenOperation extends NewBetweenOperatorNode implements DorisExpression { + public DorisBetweenOperation(DorisExpression left, DorisExpression middle, DorisExpression right, boolean isTrue) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(middle), DorisExprToNode.cast(right), isTrue); + } + + public DorisExpression getLeftExpr() { + return (DorisExpression) left; + } + + public DorisExpression getMiddleExpr() { + return (DorisExpression) middle; + } + + public DorisExpression getRightExpr() { + return (DorisExpression) right; + } + + @Override + public DorisConstant getExpectedValue() { + DorisBinaryComparisonOperation leftComparison = new DorisBinaryComparisonOperation(getMiddleExpr(), + getLeftExpr(), DorisBinaryComparisonOperation.DorisBinaryComparisonOperator.LESS_EQUALS); + DorisBinaryComparisonOperation rightComparison = new DorisBinaryComparisonOperation(getLeftExpr(), + getRightExpr(), DorisBinaryComparisonOperation.DorisBinaryComparisonOperator.LESS_EQUALS); + return new DorisBinaryLogicalOperation(leftComparison, rightComparison, + DorisBinaryLogicalOperation.DorisBinaryLogicalOperator.AND).getExpectedValue(); + } + + @Override + public DorisSchema.DorisDataType getExpectedType() { + return DorisSchema.DorisDataType.BOOLEAN; + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java new file mode 100644 index 000000000..244404569 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java @@ -0,0 +1,143 @@ +package sqlancer.doris.ast; + +import java.util.function.BinaryOperator; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisBinaryArithmeticOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisBinaryArithmeticOperation(DorisExpression left, DorisExpression right, BinaryOperatorNode.Operator op) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + } + + public enum DorisBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + ADDITION("+") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> l + r); + } + }, + SUBTRACTION("-") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> l - r); + } + }, + MULTIPLICATION("*") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> l * r); + } + }, + DIVISION("/") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l / r); + } + }, + MODULO("%") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return applyOperation(left, right, (l, r) -> r == 0 ? -1 : l % r); + } + }, + CONCAT("||") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isBoolean() || !right.isBoolean()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> l == 1 || r == 1 ? 1.0 : 0.0); + } + }, + BIT_AND("&") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() & (int) r.doubleValue())); + } + }, + BIT_OR("|") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() | (int) r.doubleValue())); + } + }, + LSHIFT("<<") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() << (int) r.doubleValue())); + } + }, + RSHIFT(">>") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (!left.isInt() || !right.isInt()) { + return DorisConstant.createNullConstant(); + } + return applyOperation(left, right, (l, r) -> (double) ((int) l.doubleValue() >> (int) r.doubleValue())); + } + }; + + private final String textRepresentation; + + DorisBinaryArithmeticOperator(String text) { + textRepresentation = text; + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + public DorisConstant applyOperation(DorisConstant left, DorisConstant right, BinaryOperator op) { + if (left.isNull() || right.isNull()) { + return DorisConstant.createNullConstant(); + } + double leftVal = left.cast(DorisDataType.FLOAT).asFloat(); + double rightVal = right.cast(DorisDataType.FLOAT).asFloat(); + return DorisConstant.createFloatConstant(op.apply(leftVal, rightVal)); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + + public DorisExpression getLeftExpr() { + return (DorisExpression) super.getLeft(); + } + + public DorisExpression getRightExpr() { + return (DorisExpression) super.getRight(); + } + + public DorisBinaryArithmeticOperator getOp() { + return (DorisBinaryArithmeticOperator) op; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftValue = getLeftExpr().getExpectedValue(); + DorisConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.FLOAT; + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java new file mode 100644 index 000000000..894d2ab9b --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java @@ -0,0 +1,120 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisBinaryComparisonOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisBinaryComparisonOperation(DorisExpression left, DorisExpression right, + DorisBinaryComparisonOperator op) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + } + + public DorisExpression getLeftExpression() { + return (DorisExpression) super.getLeft(); + } + + public DorisExpression getRightExpression() { + return (DorisExpression) super.getRight(); + } + + public DorisBinaryComparisonOperator getOp() { + return (DorisBinaryComparisonOperator) op; + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftExpectedValue = getLeftExpression().getExpectedValue(); + DorisConstant rightExpectedValue = getRightExpression().getExpectedValue(); + if (leftExpectedValue == null || rightExpectedValue == null) { + return null; + } + return getOp().apply(leftExpectedValue, rightExpectedValue); + } + + public enum DorisBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return left.valueEquals(right); + } + }, + NOT_EQUALS("!=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean()) { + return DorisConstant.createBooleanConstant(!valueEquals.asBoolean()); + } + // maybe DorisNULLConstant or null object + return valueEquals; + } + }, + LESS("<") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + return left.valueLessThan(right); + } + }, + LESS_EQUALS("<=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueLessThan = left.valueLessThan(right); + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean() && valueEquals.asBoolean()) { + return valueEquals; + } + return valueLessThan; + } + }, + GREATER(">") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueLessThan = left.valueLessThan(right); + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean() && valueEquals.asBoolean()) { + return DorisConstant.createBooleanConstant(false); + } + if (valueLessThan.isNull()) { + return valueLessThan; + } + return DorisConstant.createBooleanConstant(!valueLessThan.asBoolean()); + } + }, + GREATER_EQUALS(">=") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant valueLessThan = left.valueLessThan(right); + DorisConstant valueEquals = left.valueEquals(right); + if (valueEquals.isBoolean() && valueEquals.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } + if (valueLessThan.isNull()) { + return valueLessThan; + } + return DorisConstant.createBooleanConstant(!valueLessThan.asBoolean()); + } + }; + + private final String textRepresentation; + + DorisBinaryComparisonOperator(String text) { + textRepresentation = text; + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + } + +} diff --git a/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java new file mode 100644 index 000000000..b62ccd61b --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java @@ -0,0 +1,130 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisBinaryLogicalOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisBinaryLogicalOperation(DorisExpression left, DorisExpression right, DorisBinaryLogicalOperator op) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + } + + public DorisExpression getLeftExpr() { + return (DorisExpression) super.getLeft(); + } + + public DorisExpression getRightExpr() { + return (DorisExpression) super.getRight(); + } + + public DorisBinaryLogicalOperator getOp() { + return (DorisBinaryLogicalOperator) op; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftValue = getLeftExpr().getExpectedValue(); + DorisConstant rightValue = getRightExpr().getExpectedValue(); + if (leftValue == null || rightValue == null) { + return null; + } + return getOp().apply(leftValue, rightValue); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + public enum DorisBinaryLogicalOperator implements BinaryOperatorNode.Operator { + /* + * null and false -> false null and true -> null null or false -> null null or true -> true + */ + AND("AND", "and") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant leftVal = left.cast(DorisDataType.BOOLEAN); + DorisConstant rightVal = right.cast(DorisDataType.BOOLEAN); + assert leftVal.isNull() || leftVal.isBoolean() : leftVal + "is not null or boolean"; + assert rightVal.isNull() || rightVal.isBoolean() : rightVal + "is not null or boolean"; + if (leftVal.isNull() && rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (leftVal.isNull()) { + if (!rightVal.asBoolean()) { + return DorisConstant.createBooleanConstant(false); + } else { + return DorisConstant.createNullConstant(); + } + } + if (rightVal.isNull()) { + if (!leftVal.asBoolean()) { + return DorisConstant.createBooleanConstant(false); + } else { + return DorisConstant.createNullConstant(); + } + } + if (leftVal.asBoolean() && right.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } + return DorisConstant.createBooleanConstant(false); + } + }, + OR("OR", "or") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + DorisConstant leftVal = left.cast(DorisDataType.BOOLEAN); + DorisConstant rightVal = right.cast(DorisDataType.BOOLEAN); + assert leftVal.isNull() || leftVal.isBoolean() : leftVal + "is not null or boolean"; + assert rightVal.isNull() || rightVal.isBoolean() : rightVal + "is not null or boolean"; + if (leftVal.isNull() && rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (leftVal.isNull()) { + if (rightVal.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } else { + return DorisConstant.createNullConstant(); + } + } + if (rightVal.isNull()) { + if (leftVal.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } else { + return DorisConstant.createNullConstant(); + } + } + if (leftVal.asBoolean() || right.asBoolean()) { + return DorisConstant.createBooleanConstant(true); + } + return DorisConstant.createBooleanConstant(false); + } + }; + + private final String[] textRepresentations; + + DorisBinaryLogicalOperator(String... textRepresentations) { + this.textRepresentations = textRepresentations.clone(); + } + + @Override + public String getTextRepresentation() { + return Randomly.fromOptions(textRepresentations); + } + + public DorisBinaryLogicalOperator getRandomOp() { + return Randomly.fromOptions(values()); + } + + public static DorisBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + } + +} diff --git a/src/sqlancer/doris/ast/DorisCaseOperation.java b/src/sqlancer/doris/ast/DorisCaseOperation.java new file mode 100644 index 000000000..e852e36bd --- /dev/null +++ b/src/sqlancer/doris/ast/DorisCaseOperation.java @@ -0,0 +1,38 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.Node; + +public class DorisCaseOperation implements Node, DorisExpression { + + private final DorisExpression expr; + private final List conditions; + private final List thenClauses; + private final DorisExpression elseClause; + + public DorisCaseOperation(DorisExpression expr, List conditions, List thenClauses, + DorisExpression elseClause) { + this.expr = expr; + this.conditions = conditions; + this.thenClauses = thenClauses; + this.elseClause = elseClause; + } + + public DorisExpression getExpr() { + return expr; + } + + public List getConditions() { + return conditions; + } + + public List getThenClauses() { + return thenClauses; + } + + public DorisExpression getElseClause() { + return elseClause; + } + +} diff --git a/src/sqlancer/doris/ast/DorisCastOperation.java b/src/sqlancer/doris/ast/DorisCastOperation.java new file mode 100644 index 000000000..1aacdaf06 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisCastOperation.java @@ -0,0 +1,46 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisCastOperation implements Node, DorisExpression { + Node expr; + DorisDataType type; + + public DorisCastOperation(Node expr, DorisCompositeDataType type) { + this.expr = expr; + this.type = type.getPrimitiveDataType(); + } + + public DorisCastOperation(Node expr, DorisDataType type) { + this.expr = expr; + this.type = type; + } + + public Node getExpr() { + return expr; + } + + public DorisExpression getExpression() { + return (DorisExpression) expr; + } + + public DorisDataType getType() { + return type; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return expectedValue.cast(type); + } + + @Override + public DorisDataType getExpectedType() { + return type; + } +} diff --git a/src/sqlancer/doris/ast/DorisColumnValue.java b/src/sqlancer/doris/ast/DorisColumnValue.java new file mode 100644 index 000000000..fe3ec1448 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisColumnValue.java @@ -0,0 +1,53 @@ +package sqlancer.doris.ast; + +import java.util.Objects; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisDataType; + +public class DorisColumnValue extends ColumnReferenceNode implements DorisExpression { + + private final DorisConstant expectedValue; + + public DorisColumnValue(DorisColumn column, DorisConstant value) { + super(column); + this.expectedValue = value; + } + + @Override + public DorisConstant getExpectedValue() { + return expectedValue; + } + + @Override + public DorisDataType getExpectedType() { + return getColumn().getType().getPrimitiveDataType(); + } + + public static DorisColumnValue create(DorisColumn column, DorisConstant value) { + return new DorisColumnValue(column, value); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + DorisColumnValue that = (DorisColumnValue) o; + if (!this.getColumn().getName().equals(that.getColumn().getName())) { + return false; + } + return Objects.equals(expectedValue, that.expectedValue); + } + + @Override + public int hashCode() { + String nameAndValue = this.getColumn().getName(); + nameAndValue += expectedValue == null ? "NULL" : expectedValue.toString(); + return Objects.hash(nameAndValue); + } +} diff --git a/src/sqlancer/doris/ast/DorisConstant.java b/src/sqlancer/doris/ast/DorisConstant.java new file mode 100644 index 000000000..37374a4d7 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisConstant.java @@ -0,0 +1,681 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.utils.DorisNumberUtils; + +public abstract class DorisConstant implements Node, DorisExpression { + + private DorisConstant() { + } + + public boolean isNull() { + return false; + } + + public boolean isInt() { + return false; + } + + public boolean isBoolean() { + return false; + } + + public boolean isString() { + return false; + } + + public boolean isFloat() { + return false; + } + + public boolean isNum() { + // for INT, FLOAT, BOOLEAN + return false; + } + + public boolean isDate() { + return false; + } + + public boolean isDatetime() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asFloat() { + throw new UnsupportedOperationException(this.toString()); + } + + public abstract DorisConstant cast(DorisDataType dataType); + + public abstract DorisConstant valueEquals(DorisConstant rightVal); + + public abstract DorisConstant valueLessThan(DorisConstant rightVal); + + public static class DorisNullConstant extends DorisConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + return DorisConstant.createNullConstant(); + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + return DorisConstant.createNullConstant(); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + return DorisConstant.createNullConstant(); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.NULL; + } + } + + public static class DorisIntConstant extends DorisConstant { + + private final long value; + + public DorisIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + @Override + public boolean isInt() { + return true; + } + + @Override + public boolean isNum() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + return this; + case FLOAT: + case DECIMAL: + return new DorisFloatConstant(value); + case VARCHAR: + return new DorisTextConstant(String.valueOf(value)); + case BOOLEAN: + return new DorisBooleanConstant(value != 0); + default: + return DorisConstant.createNullConstant(); + } + } + + @Override + public long asInt() { + return value; + } + + @Override + public boolean asBoolean() { + return value != 0; + } + + @Override + public double asFloat() { + return value; + } + + @Override + public String asString() { + return String.valueOf(value); + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isNum()) { + return DorisConstant.createBooleanConstant(value == rightVal.asFloat()); + } + + throw new AssertionError(rightVal); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isNum()) { + return DorisConstant.createBooleanConstant(value < rightVal.asFloat()); + } + + throw new AssertionError(rightVal); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.INT; + } + + } + + public static class DorisFloatConstant extends DorisConstant { + + private final double value; + + public DorisFloatConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public boolean isFloat() { + return true; + } + + @Override + public boolean isNum() { + return true; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "3.40282347e+38"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-3.40282347e+38"; + } + + return String.valueOf(value); + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + return new DorisIntConstant((long) value); + case FLOAT: + case DECIMAL: + return this; + case VARCHAR: + return new DorisTextConstant(String.valueOf(value)); + case BOOLEAN: + return new DorisBooleanConstant(value >= 1); + default: + return null; + } + } + + @Override + public double asFloat() { + return value; + } + + @Override + public String asString() { + return toString(); + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DorisConstant.createBooleanConstant(value == rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DorisConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } else if (rightVal.isInt()) { + return DorisConstant.createBooleanConstant(value < rightVal.asInt()); + } else if (rightVal.isFloat()) { + return DorisConstant.createBooleanConstant(value < rightVal.asFloat()); + } else { + throw new AssertionError(rightVal); + } + } + + } + + public static class DorisTextConstant extends DorisConstant { + + private final String value; + + public DorisTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; + } + + @Override + public String asString() { + return value; + } + + @Override + public boolean isString() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + // Currently only supports conversion of int text to int, not float text, see + // https://github.com/apache/doris/issues/18227 + if (DorisNumberUtils.isNumber(value)) { + long val = (long) Double.parseDouble(value); + return new DorisIntConstant(val); + } + return new DorisNullConstant(); + case FLOAT: + case DECIMAL: + if (DorisNumberUtils.isNumber(value)) { + return new DorisFloatConstant(Double.parseDouble(value)); + } + return new DorisNullConstant(); + case DATE: + if (DorisNumberUtils.isDate(value)) { + return new DorisDateConstant(value); + } + return new DorisNullConstant(); + case DATETIME: + if (DorisNumberUtils.isDatetime(value)) { + return new DorisDatetimeConstant(value); + } + return new DorisNullConstant(); + case VARCHAR: + return this; + case BOOLEAN: + if ("false".contentEquals(value.toLowerCase())) { + return new DorisBooleanConstant(false); + } + if ("true".contentEquals(value.toLowerCase())) { + return new DorisBooleanConstant(true); + } + return new DorisNullConstant(); + default: + return new DorisNullConstant(); + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isString()) { + return DorisConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); + } + if (DorisNumberUtils.isNumber(value) && rightVal.isNum()) { + return DorisConstant.createBooleanConstant(Double.parseDouble(value) == rightVal.asFloat()); + } + // Doris currently does not support judgment between string types and other types, such date, datetime + return DorisConstant.createBooleanConstant(false); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isString()) { + return DorisConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); + } + if (DorisNumberUtils.isNumber(value) && rightVal.isNum()) { + return DorisConstant.createBooleanConstant(Double.parseDouble(value) < rightVal.asFloat()); + } + // Doris currently does not support judgment between string types and other types, such date, datetime + return DorisConstant.createBooleanConstant(false); + } + + } + + public static class DorisDateConstant extends DorisConstant { + + public String textRepr; + + public DorisDateConstant(long val) { + textRepr = DorisNumberUtils.timestampToDateText(val); + } + + public DorisDateConstant(String textRepr) { + this.textRepr = textRepr; + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + @Override + public String asString() { + return textRepr; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case VARCHAR: + return new DorisTextConstant(textRepr); + case DATE: + return this; + case DATETIME: + return new DorisDatetimeConstant(DorisNumberUtils.dateTextToDatetimeText(textRepr)); + default: + return new DorisNullConstant(); + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && rightVal.asString().contentEquals("CURRENT_TIMESTAMP")) { + return DorisConstant.createBooleanConstant(false); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant.createBooleanConstant(DorisNumberUtils.dateEqual(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && rightVal.asString().contentEquals("CURRENT_TIMESTAMP")) { + return DorisConstant.createBooleanConstant( + DorisNumberUtils.dateLessThan(textRepr, DorisNumberUtils.getCurrentTimeText())); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant + .createBooleanConstant(DorisNumberUtils.dateLessThan(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public boolean isDate() { + return true; + } + } + + public static class DorisDatetimeConstant extends DorisConstant { + + public String textRepr; + + public DorisDatetimeConstant(long val) { + textRepr = DorisNumberUtils.timestampToDatetimeText(val); + } + + public DorisDatetimeConstant(String textRepr) { + this.textRepr = textRepr; + } + + public DorisDatetimeConstant() { + textRepr = "CURRENT_TIMESTAMP"; + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + @Override + public String asString() { + return textRepr; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case VARCHAR: + return new DorisTextConstant(textRepr); + case DATE: + return new DorisDatetimeConstant(DorisNumberUtils.datetimeTextToDateText(textRepr)); + case DATETIME: + return this; + default: + return new DorisNullConstant(); + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && (rightVal.asString().contentEquals("CURRENT_TIMESTAMP") + || textRepr.contentEquals("CURRENT_TIMESTAMP"))) { + boolean isEq = rightVal.asString().contentEquals("CURRENT_TIMESTAMP") + && textRepr.contentEquals("CURRENT_TIMESTAMP"); + return DorisConstant.createBooleanConstant(isEq); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant + .createBooleanConstant(DorisNumberUtils.datetimeEqual(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isDatetime() && (rightVal.asString().contentEquals("CURRENT_TIMESTAMP") + || textRepr.contentEquals("CURRENT_TIMESTAMP"))) { + String leftText = textRepr; + String rightText = rightVal.asString(); + if (leftText.contentEquals(rightText)) { + return DorisConstant.createBooleanConstant(false); + } + if (leftText.contentEquals("CURRENT_TIMESTAMP")) { + leftText = DorisNumberUtils.getCurrentTimeText(); + } + if (rightText.contentEquals("CURRENT_TIMESTAMP")) { + rightText = DorisNumberUtils.getCurrentTimeText(); + } + boolean lessThan = DorisNumberUtils.dateLessThan(leftText, rightText); + return DorisConstant.createBooleanConstant(lessThan); + } + if (rightVal.isString() || rightVal.isDate() || rightVal.isDatetime()) { + return DorisConstant + .createBooleanConstant(DorisNumberUtils.datetimeLessThan(textRepr, rightVal.asString())); + } + return DorisConstant.createBooleanConstant(false); + } + + @Override + public boolean isDatetime() { + return true; + } + + } + + public static class DorisBooleanConstant extends DorisConstant { + + private final boolean value; + + public DorisBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public String asString() { + return toString(); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + @Override + public boolean isNum() { + return true; + } + + @Override + public DorisConstant cast(DorisDataType dataType) { + switch (dataType) { + case INT: + return new DorisIntConstant(value ? 1 : 0); + case FLOAT: + case DECIMAL: + return new DorisFloatConstant(value ? 1 : 0); + case BOOLEAN: + return this; + case VARCHAR: + return new DorisTextConstant(value ? "1" : "0"); + default: + return null; + } + } + + @Override + public DorisConstant valueEquals(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isBoolean()) { + return DorisConstant.createBooleanConstant(value == rightVal.asBoolean()); + } + if (rightVal.isNum() || rightVal.isString() && DorisNumberUtils.isNumber(rightVal.asString())) { + return DorisConstant.createBooleanConstant((value ? 1 : 0) == rightVal.asFloat()); + } + throw new AssertionError(rightVal); + } + + @Override + public DorisConstant valueLessThan(DorisConstant rightVal) { + if (rightVal.isNull()) { + return DorisConstant.createNullConstant(); + } + if (rightVal.isBoolean()) { + return DorisConstant.createBooleanConstant(value == rightVal.asBoolean()); + } + if (rightVal.isNum() || rightVal.isString() && DorisNumberUtils.isNumber(rightVal.asString())) { + return DorisConstant.createBooleanConstant((value ? 1 : 0) == rightVal.asFloat()); + } + throw new AssertionError(rightVal); + } + } + + public static DorisConstant createStringConstant(String text) { + return new DorisTextConstant(text); + } + + public static DorisConstant createFloatConstant(double val) { + return new DorisFloatConstant(val); + } + + public static DorisConstant createIntConstant(long val) { + return new DorisIntConstant(val); + } + + public static DorisConstant createNullConstant() { + return new DorisNullConstant(); + } + + public static DorisConstant createBooleanConstant(boolean val) { + return new DorisBooleanConstant(val); + } + + public static DorisConstant createDateConstant(long integer) { + return new DorisDateConstant(integer); + } + + public static DorisConstant createDateConstant(String date) { + return new DorisDateConstant(date); + } + + public static DorisConstant createDatetimeConstant(long integer) { + return new DorisDatetimeConstant(integer); + } + + public static DorisConstant createDatetimeConstant(String datetime) { + return new DorisDatetimeConstant(datetime); + } + + public static DorisConstant createDatetimeConstant() { + // use CURRENT_TIMESTAMP + return new DorisDatetimeConstant(); + } + +} diff --git a/src/sqlancer/doris/ast/DorisExpression.java b/src/sqlancer/doris/ast/DorisExpression.java new file mode 100644 index 000000000..559ecd8f3 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisExpression.java @@ -0,0 +1,13 @@ +package sqlancer.doris.ast; + +import sqlancer.doris.DorisSchema; + +public interface DorisExpression { + default DorisSchema.DorisDataType getExpectedType() { + return null; + } + + default DorisConstant getExpectedValue() { + return null; + } +} diff --git a/src/sqlancer/doris/ast/DorisFunctionOperation.java b/src/sqlancer/doris/ast/DorisFunctionOperation.java new file mode 100644 index 000000000..0a75892d0 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisFunctionOperation.java @@ -0,0 +1,282 @@ +package sqlancer.doris.ast; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.gen.DorisNewExpressionGenerator; + +public class DorisFunctionOperation implements Node, DorisExpression { + + private DorisFunction function; + private List args; + + // https://doris.apache.org/zh-CN/docs/dev/summary/basic-summary + public enum DorisFunction { + + // Array functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/array-functions/array + // Skip now + + // Date functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/date-time-functions/convert_tz/ + CONVERT_TZ(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CURDATE(false, DorisDataType.DATE), CURRENT_DATE(false, DorisDataType.DATE), + CURTIME(false, DorisDataType.VARCHAR), CURRENT_TIME(false, DorisDataType.VARCHAR), + CURRENT_TIMESTAMP(false, DorisDataType.DATETIME), LOCALTIME(false, DorisDataType.DATETIME), + LOCALTIMESTAMP(false, DorisDataType.DATETIME), NOW(false, DorisDataType.DATETIME), + YEAR(false, DorisDataType.INT, DorisDataType.DATETIME), + QUARTER(false, DorisDataType.INT, DorisDataType.DATETIME), + MONTH(false, DorisDataType.INT, DorisDataType.DATETIME), DAY(false, DorisDataType.INT, DorisDataType.DATETIME), + DAYOFYEAR(false, DorisDataType.INT, DorisDataType.DATETIME), + DAYOFMONTH(false, DorisDataType.INT, DorisDataType.DATETIME), + DAYOFWEEK(false, DorisDataType.INT, DorisDataType.DATETIME), WEEK(false, DorisDataType.INT, DorisDataType.DATE), + WEEKDAY(false, DorisDataType.INT, DorisDataType.DATE), + WEEKOFYEAR(false, DorisDataType.INT, DorisDataType.DATETIME), + YEARWEEK(false, DorisDataType.INT, DorisDataType.DATE), + DAYNAME(false, DorisDataType.VARCHAR, DorisDataType.DATETIME), + MONTHNAME(false, DorisDataType.VARCHAR, DorisDataType.DATETIME), + HOUR(false, DorisDataType.INT, DorisDataType.DATETIME), + MINUTE(false, DorisDataType.INT, DorisDataType.DATETIME), + SECOND(false, DorisDataType.INT, DorisDataType.DATETIME), + FROM_DAYS(false, DorisDataType.DATE, DorisDataType.INT), + LAST_DAYS(false, DorisDataType.DATE, DorisDataType.DATETIME), + TO_MONDAY(false, DorisDataType.DATE, DorisDataType.DATETIME), + FROM_UNIXTIME(false, DorisDataType.DATETIME, DorisDataType.INT), + UNIX_TIMESTAMP(false, DorisDataType.INT, DorisDataType.DATETIME), UTC_TIMESTAMP(false, DorisDataType.DATETIME), + TO_DATE(false, DorisDataType.DATE, DorisDataType.DATETIME), + TO_DAYS(false, DorisDataType.INT, DorisDataType.DATETIME), + TIME_TO_SEC(false, DorisDataType.INT, DorisDataType.DATETIME), + // EXTRACT(1), // select extract(year from '2022-09-22 17:01:30') as year, currently not considered + MAKEDATE(false, DorisDataType.DATE, DorisDataType.INT, DorisDataType.INT), + STR_TO_DATE(false, DorisDataType.DATETIME, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + TIME_ROUND(false, DorisDataType.DATETIME, DorisDataType.DATETIME), + TIME_DIFF(false, DorisDataType.VARCHAR, DorisDataType.DATETIME, DorisDataType.DATETIME), + TIMESTAMPADD(false, DorisDataType.DATETIME, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.DATETIME), + TIMESTAMPDIFF(false, DorisDataType.VARCHAR, DorisDataType.DATETIME, DorisDataType.DATETIME), + DATE_ADD(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATE_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATE_TRUNC(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATE_FORMAT(false, DorisDataType.VARCHAR, DorisDataType.DATETIME, DorisDataType.VARCHAR), + DATEDIFF(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.DATETIME), + // MICROSECONDS_ADD(false), + MINUTES_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + MINUTES_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + MINUTES_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + SECONDS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + SECONDS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + SECONDS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + HOURS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + HOURS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + HOURS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + DAYS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + DAYS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + DAYS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + WEEKS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + WEEKS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + WEEKS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + MONTHS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + MONTHS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + MONTHS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + YEARS_ADD(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + YEARS_DIFF(false, DorisDataType.INT, DorisDataType.DATETIME, DorisDataType.DATETIME), + YEARS_SUB(false, DorisDataType.DATETIME, DorisDataType.DATETIME, DorisDataType.INT), + + // GIS functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/spatial-functions/st_x + // Skip now + + // String functions, https://doris.apache.org/docs/dev/sql-manual/sql-functions/string-functions/to_base64 + TO_BASE64(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + FROM_BASE64(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + ASCII(false, DorisDataType.INT, DorisDataType.VARCHAR), LENGTH(false, DorisDataType.INT, DorisDataType.VARCHAR), + BIT_LENGTH(false, DorisDataType.INT, DorisDataType.VARCHAR), + CHAR_LENGTH(false, DorisDataType.INT, DorisDataType.VARCHAR), + LPAD(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.VARCHAR), + RPAD(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.VARCHAR), + LOWER(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LCASE(false, DorisDataType.INT, DorisDataType.VARCHAR), + UPPER(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + UCASE(false, DorisDataType.INT, DorisDataType.VARCHAR), + INITCAP(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + REPEAT(false, DorisDataType.VARCHAR, DorisDataType.INT), + REVERSE(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CHAR(true, DorisDataType.VARCHAR, DorisDataType.INT), + CONCAT(true, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CONCAT_WS(true, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + SUBSTR(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.INT), + SUBSTRING(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + SUB_REPLACE(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + APPEND_TRAILING_CHAR_IF_ABSENT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + ENDS_WITH(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + STARTS_WITH(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + TRIM(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LTRIM(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + RTRIM(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + NULL_OR_EMPTY(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR), + NOT_NULL_OR_EMPTY(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR), + HEX(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + UNHEX(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + ELT(true, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.VARCHAR), + INSTR(false, DorisDataType.INT, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LOCATE(false, DorisDataType.INT, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + // FIELD(1, true), + FIND_IN_SET(false, DorisDataType.INT, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + REPLACE(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + LEFT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + RIGHT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + STRLEFT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + STRRIGHT(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + SPLIT_PART(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + // SPLIT_BY_STRING(2), + SUBSTRING_INDEX(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.INT), + MONEY_FORMAT(false, DorisDataType.VARCHAR, DorisDataType.DECIMAL), + PARSE_URL(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + CONVERT_TO(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + EXTRACT_URL_PARAMETER(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + UUID(false, DorisDataType.VARCHAR), SPACE(false, DorisDataType.VARCHAR, DorisDataType.INT), + // SLEEP(1), + ESQUERY(false, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + MASK(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + MASK_FIRST_N(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + MASK_LAST_N(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + // MULTI_SEARCH_ALL_POSITIONS(2), + // MULTI_MATCH_ANY(2), + + // BITMAP functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/bitmap-functions/to_bitmap + // skip now + + // Bitwise functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/bitwise-functions/bitand + BITAND(false, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BITOR(false, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BITXOR(false, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BITNOT(false, DorisDataType.INT, DorisDataType.INT), + + // condition funtions + // case(), + COALESCE(true, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + IF(false, DorisDataType.VARCHAR, DorisDataType.BOOLEAN, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + IFNULL(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + NVL(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + NULLIF(false, DorisDataType.VARCHAR, DorisDataType.VARCHAR, DorisDataType.VARCHAR), + + // JSON Functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/json-functions/jsonb_parse + // skip now + + // Hash functions, + // https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/hash-functions/murmur_hash3_32 + MURMUR_HASH3_32(true, DorisDataType.INT, DorisDataType.VARCHAR), + MURMUR_HASH3_64(true, DorisDataType.INT, DorisDataType.VARCHAR), + + // HLL functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/hll-functions/hll_cardinality + // skip now + + // Math functions, https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/math-functions/conv + CONV(false, DorisDataType.VARCHAR, DorisDataType.INT, DorisDataType.INT, DorisDataType.INT), + BIN(false, DorisDataType.VARCHAR, DorisDataType.INT), SIN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + COS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), TAN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + ASIN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), ACOS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + ATAN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), E(false, DorisDataType.FLOAT), + PI(false, DorisDataType.FLOAT), EXP(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + LOG(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT), + LOG2(false, DorisDataType.FLOAT, DorisDataType.FLOAT), LN(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + LOG10(false, DorisDataType.FLOAT, DorisDataType.FLOAT), CEIL(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + FLOOR(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + PMOD(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT), + ROUND(false, DorisDataType.INT, DorisDataType.FLOAT), + ROUND_BANKERS(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.INT), + TRUNCATE(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.INT), + ABS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), SQRT(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + CBRT(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + POW(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT), + DEGREES(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + RADIANS(false, DorisDataType.FLOAT, DorisDataType.FLOAT), SIGN(false, DorisDataType.INT, DorisDataType.FLOAT), + POSTIVE(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + NEGATIVE(false, DorisDataType.FLOAT, DorisDataType.FLOAT), + GREATEST(true, DorisDataType.FLOAT, DorisDataType.FLOAT), LEAST(true, DorisDataType.FLOAT, DorisDataType.FLOAT), + RANDOM(false, DorisDataType.FLOAT), MOD(false, DorisDataType.FLOAT, DorisDataType.FLOAT, DorisDataType.FLOAT); + + // encrypt-digest-functions, + // https://doris.apache.org/zh-CN/docs/dev/sql-manual/sql-functions/encrypt-digest-functions/aes + // skip now + + private boolean isVariadic; // If isVALid is true, then treat the last argumentTypes as an infinite type + private DorisDataType returnType; + private DorisDataType[] argumentTypes; + private String functionName; + + DorisFunction(String functionName, boolean isVariadic, DorisDataType returnType, + DorisDataType... argumentTypes) { + this.functionName = functionName; + this.isVariadic = isVariadic; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + DorisFunction(boolean isVariadic, DorisDataType returnType, DorisDataType... argumentTypes) { + this.functionName = toString(); + this.isVariadic = isVariadic; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + DorisFunction(boolean isVariadic, DorisDataType returnType) { + this.functionName = toString(); + this.isVariadic = isVariadic; + this.returnType = returnType; + this.argumentTypes = null; + } + + public String getFunctionName() { + return functionName; + } + + public static DorisFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public boolean isVariadic() { + return isVariadic; + } + + public boolean isCompatibleWithReturnType(DorisDataType returnType) { + return this.returnType == returnType; + } + + public DorisDataType[] getArgumentTypes() { + if (argumentTypes == null) { + return null; + } + return argumentTypes.clone(); + } + + public DorisFunctionOperation getCall(DorisDataType returnType, DorisNewExpressionGenerator gen, int depth) { + List arguments = new ArrayList<>(); + if (getArgumentTypes() != null) { + Stream.of(getArgumentTypes()).forEach(arg -> arguments.add(gen.generateExpression(arg, depth + 1))); + } + return new DorisFunctionOperation(this, arguments); + } + + public static List getFunctionsCompatibleWith(DorisDataType returnType) { + return Stream.of(values()).filter(f -> f.isCompatibleWithReturnType(returnType)) + .collect(Collectors.toList()); + } + + } + + public DorisFunctionOperation(DorisFunction function, List args) { + this.function = function; + this.args = args; + } + + public List getArgs() { + return args; + } + + public DorisFunction getFunction() { + return function; + } + +} diff --git a/src/sqlancer/doris/ast/DorisInOperation.java b/src/sqlancer/doris/ast/DorisInOperation.java new file mode 100644 index 000000000..31dcbe1ec --- /dev/null +++ b/src/sqlancer/doris/ast/DorisInOperation.java @@ -0,0 +1,53 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisInOperation extends NewInOperatorNode implements DorisExpression { + + private final DorisExpression leftExpr; + private final List rightExpr; + + public DorisInOperation(DorisExpression left, List right, boolean isNegated) { + super(DorisExprToNode.cast(left), DorisExprToNode.casts(right), isNegated); + this.leftExpr = left; + this.rightExpr = right; + } + + @Override + public DorisSchema.DorisDataType getExpectedType() { + return DorisSchema.DorisDataType.BOOLEAN; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftValue = leftExpr.getExpectedValue(); + if (leftValue == null) { + return null; + } + if (leftValue.isNull()) { + return DorisConstant.createNullConstant(); + } + boolean containNull = false; + for (DorisExpression expr : rightExpr) { + DorisConstant rightValue = expr.getExpectedValue(); + if (rightValue == null) { + return null; + } + if (rightValue.isNull()) { + containNull = true; + } else if (rightValue.valueEquals(leftValue).isBoolean() && rightValue.valueEquals(leftValue).asBoolean()) { + return DorisConstant.createBooleanConstant(!isNegated()); + } + } + + if (containNull) { + return DorisConstant.createNullConstant(); + } + // should return false when not considering isNegated op + return DorisConstant.createBooleanConstant(isNegated()); + } +} diff --git a/src/sqlancer/doris/ast/DorisJoin.java b/src/sqlancer/doris/ast/DorisJoin.java new file mode 100644 index 000000000..01274449f --- /dev/null +++ b/src/sqlancer/doris/ast/DorisJoin.java @@ -0,0 +1,108 @@ +package sqlancer.doris.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + + public enum JoinType { + INNER, STRAIGHT, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public DorisJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + public static List> getJoins(List> tableList, + DorisGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + DorisNewExpressionGenerator joinGen = new DorisNewExpressionGenerator(globalState).setColumns(columns); + switch (DorisJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(DorisJoin.createInnerJoin(leftTable, rightTable, + DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + break; + case STRAIGHT: + joinExpressions.add(DorisJoin.createStraightJoin(leftTable, rightTable, + DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + break; + case LEFT: + joinExpressions.add(DorisJoin.createLeftOuterJoin(leftTable, rightTable, + DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + break; + case RIGHT: + joinExpressions.add(DorisJoin.createRightOuterJoin(leftTable, rightTable, + DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static DorisJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DorisJoin(left, right, JoinType.INNER, predicate); + } + + public static DorisJoin createStraightJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DorisJoin(left, right, JoinType.STRAIGHT, predicate); + } + + public static DorisJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DorisJoin(left, right, JoinType.RIGHT, predicate); + } + + public static DorisJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DorisJoin(left, right, JoinType.LEFT, predicate); + } +} diff --git a/src/sqlancer/doris/ast/DorisLikeOperation.java b/src/sqlancer/doris/ast/DorisLikeOperation.java new file mode 100644 index 000000000..a3251ca7f --- /dev/null +++ b/src/sqlancer/doris/ast/DorisLikeOperation.java @@ -0,0 +1,85 @@ +package sqlancer.doris.ast; + +import sqlancer.LikeImplementationHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisLikeOperation extends NewBinaryOperatorNode implements DorisExpression { + + public DorisLikeOperation(DorisExpression left, DorisExpression right, DorisLikeOperator op) { + super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + public DorisExpression getLeftExpr() { + return (DorisExpression) super.getLeft(); + } + + public DorisExpression getRightExpr() { + return (DorisExpression) super.getRight(); + } + + public DorisLikeOperator getOp() { + return (DorisLikeOperator) op; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant leftVal = getLeftExpr().getExpectedValue(); + DorisConstant rightVal = getRightExpr().getExpectedValue(); + if (leftVal == null || rightVal == null) { + return null; + } + return getOp().apply(leftVal, rightVal); + } + + public enum DorisLikeOperator implements BinaryOperatorNode.Operator { + LIKE_OPERATOR("LIKE", "like") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (left == null || right == null) { + return null; + } + if (left.isNull() || right.isNull()) { + return DorisConstant.createNullConstant(); + } + boolean result = LikeImplementationHelper.match(left.asString(), right.asString(), 0, 0, true); + return DorisConstant.createBooleanConstant(result); + } + }, + NOT_LIKE("NOT LIKE", "not like") { + @Override + public DorisConstant apply(DorisConstant left, DorisConstant right) { + if (left == null || right == null) { + return null; + } + if (left.isNull() || right.isNull()) { + return DorisConstant.createNullConstant(); + } + boolean result = LikeImplementationHelper.match(left.asString(), right.asString(), 0, 0, true); + return DorisConstant.createBooleanConstant(!result); + } + }; + + private final String[] textRepresentations; + + DorisLikeOperator(String... text) { + textRepresentations = text.clone(); + } + + public abstract DorisConstant apply(DorisConstant left, DorisConstant right); + + @Override + public String getTextRepresentation() { + return " " + Randomly.fromOptions(textRepresentations) + " "; + } + } + +} diff --git a/src/sqlancer/doris/ast/DorisOrderByTerm.java b/src/sqlancer/doris/ast/DorisOrderByTerm.java new file mode 100644 index 000000000..6207f4af2 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisOrderByTerm.java @@ -0,0 +1,10 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.Node; + +public class DorisOrderByTerm extends NewOrderingTerm implements DorisExpression { + public DorisOrderByTerm(Node expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/doris/ast/DorisSelect.java b/src/sqlancer/doris/ast/DorisSelect.java new file mode 100644 index 000000000..283d474cc --- /dev/null +++ b/src/sqlancer/doris/ast/DorisSelect.java @@ -0,0 +1,41 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class DorisSelect extends SelectBase> implements Node { + + public enum DorisSelectDistinctType { + + ALL, DISTINCT, DISTINCTROW, NULL; + + public static DorisSelectDistinctType getRandomWithoutNull() { + DorisSelectDistinctType sft; + do { + sft = Randomly.fromOptions(values()); + } while (sft == DorisSelectDistinctType.NULL); + return sft; + } + } + + private DorisSelectDistinctType selectDistinctType = DorisSelectDistinctType.ALL; + + public void setDistinct(boolean isDistinct) { + if (isDistinct) { + this.selectDistinctType = DorisSelectDistinctType.DISTINCT; + } else { + this.selectDistinctType = DorisSelectDistinctType.ALL; + } + } + + public void setDistinct(DorisSelectDistinctType type) { + this.selectDistinctType = type; + } + + public boolean isDistinct() { + return this.selectDistinctType == DorisSelectDistinctType.DISTINCT + || this.selectDistinctType == DorisSelectDistinctType.DISTINCTROW; + } + +} diff --git a/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java new file mode 100644 index 000000000..9a34e2c54 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java @@ -0,0 +1,87 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements DorisExpression { + + public DorisUnaryPostfixOperation(DorisExpression expr, DorisUnaryPostfixOperator op) { + super(DorisExprToNode.cast(expr), op); + } + + public DorisExpression getExpression() { + return (DorisExpression) getExpr(); + } + + public DorisUnaryPostfixOperator getOp() { + return (DorisUnaryPostfixOperator) op; + } + + @Override + public DorisDataType getExpectedType() { + return DorisDataType.BOOLEAN; + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } + + public enum DorisUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public DorisDataType[] getInputDataTypes() { + return DorisDataType.values(); + } + + @Override + public DorisConstant apply(DorisConstant value) { + return DorisConstant.createBooleanConstant(value.isNull()); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public DorisDataType[] getInputDataTypes() { + return DorisDataType.values(); + } + + @Override + public DorisConstant apply(DorisConstant value) { + return DorisConstant.createBooleanConstant(!value.isNull()); + } + }; + + private final String textRepresentations; + + DorisUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static DorisUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract DorisDataType[] getInputDataTypes(); + + public abstract DorisConstant apply(DorisConstant value); + } + + @Override + public String getOperatorRepresentation() { + return this.op.getTextRepresentation(); + } + +} diff --git a/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java new file mode 100644 index 000000000..e7f7d91b0 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java @@ -0,0 +1,112 @@ +package sqlancer.doris.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements DorisExpression { + + public DorisUnaryPrefixOperation(DorisExpression expr, DorisUnaryPrefixOperator op) { + super(DorisExprToNode.cast(expr), op); + } + + public DorisExpression getExpression() { + return (DorisExpression) getExpr(); + } + + public DorisUnaryPrefixOperator getOp() { + return (DorisUnaryPrefixOperator) op; + } + + @Override + public DorisDataType getExpectedType() { + return getOp().getExpressionType(getExpression()); + } + + @Override + public DorisConstant getExpectedValue() { + DorisConstant expectedValue = getExpression().getExpectedValue(); + if (expectedValue == null) { + return null; + } + return getOp().apply(expectedValue); + } + + public enum DorisUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", DorisDataType.BOOLEAN, DorisDataType.INT) { + @Override + public DorisDataType getExpressionType(DorisExpression expr) { + return DorisDataType.BOOLEAN; + } + + @Override + protected DorisConstant apply(DorisConstant value) { + if (value.isNull()) { + return DorisConstant.createNullConstant(); + } else { + return DorisConstant.createBooleanConstant(!value.cast(DorisDataType.BOOLEAN).asBoolean()); + } + } + }, + + UNARY_PLUS("+", DorisDataType.INT) { + @Override + public DorisDataType getExpressionType(DorisExpression expr) { + return expr.getExpectedType(); + } + + @Override + protected DorisConstant apply(DorisConstant value) { + return value; + } + }, + UNARY_MINUS("-", DorisDataType.INT) { + @Override + public DorisDataType getExpressionType(DorisExpression expr) { + return expr.getExpectedType(); + } + + @Override + protected DorisConstant apply(DorisConstant value) { + if (value.isNull()) { + return DorisConstant.createNullConstant(); + } + try { + if (value.isInt()) { + return DorisConstant.createIntConstant(-value.asInt()); + } + if (value.isFloat()) { + return DorisConstant.createFloatConstant(-value.asFloat()); + } + return null; + } catch (UnsupportedOperationException e) { + return null; + } + } + }; + + private String textRepresentation; + private DorisDataType[] dataTypes; + + DorisUnaryPrefixOperator(String textRepresentation, DorisDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public abstract DorisDataType getExpressionType(DorisExpression expr); + + public DorisDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); + } + + protected abstract DorisConstant apply(DorisConstant value); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + } + +} diff --git a/src/sqlancer/doris/gen/DorisAlterTableGenerator.java b/src/sqlancer/doris/gen/DorisAlterTableGenerator.java new file mode 100644 index 000000000..bfac074fc --- /dev/null +++ b/src/sqlancer/doris/gen/DorisAlterTableGenerator.java @@ -0,0 +1,50 @@ +package sqlancer.doris.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisTable; + +public final class DorisAlterTableGenerator { + + private DorisAlterTableGenerator() { + } + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(DorisCompositeDataType.getRandomWithoutNull().toString()); + break; + case ALTER_COLUMN: + sb.append("MODIFY COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" "); + sb.append(DorisCompositeDataType.getRandomWithoutNull().toString()); + break; + case DROP_COLUMN: + sb.append("DROP COLUMN "); + sb.append(table.getRandomColumn().getName()); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisDeleteGenerator.java b/src/sqlancer/doris/gen/DorisDeleteGenerator.java new file mode 100644 index 000000000..27f369aec --- /dev/null +++ b/src/sqlancer/doris/gen/DorisDeleteGenerator.java @@ -0,0 +1,31 @@ +package sqlancer.doris.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public final class DorisDeleteGenerator { + + private DorisDeleteGenerator() { + } + + public static SQLQueryAdapter generate(DorisGlobalState globalState) { + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + .setColumns(table.getColumns()).generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + DorisErrors.addExpressionErrors(errors); + } + return new SQLQueryAdapter(sb.toString(), errors); + } + +} diff --git a/src/sqlancer/doris/gen/DorisDropTableGenerator.java b/src/sqlancer/doris/gen/DorisDropTableGenerator.java new file mode 100644 index 000000000..c8bbc67d4 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisDropTableGenerator.java @@ -0,0 +1,28 @@ +package sqlancer.doris.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; + +public final class DorisDropTableGenerator { + + private DorisDropTableGenerator() { + } + + public static SQLQueryAdapter dropTable(DorisGlobalState globalState) { + if (globalState.getSchema().getTables(t -> !t.isView()).size() <= 1) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP TABLE "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> !t.isView()).getName()); + if (Randomly.getBoolean()) { + sb.append(" FORCE "); + } + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisDropViewGenerator.java b/src/sqlancer/doris/gen/DorisDropViewGenerator.java new file mode 100644 index 000000000..81c1629c2 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisDropViewGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.doris.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; + +public final class DorisDropViewGenerator { + + private DorisDropViewGenerator() { + } + + public static SQLQueryAdapter dropView(DorisGlobalState globalState) { + if (globalState.getSchema().getTables(t -> t.isView()).size() == 0) { + throw new IgnoreMeException(); + } + StringBuilder sb = new StringBuilder("DROP VIEW "); + if (Randomly.getBoolean()) { + sb.append("IF EXISTS "); + } + // TODO: DROP VIEW syntax: DROP MATERIALIZED VIEW [IF EXISTS] mv_name ON table_name; + // should record original table name in view table + sb.append(globalState.getSchema().getRandomTableOrBailout(t -> t.isView()).getName()); + return new SQLQueryAdapter(sb.toString(), null, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisIndexGenerator.java b/src/sqlancer/doris/gen/DorisIndexGenerator.java new file mode 100644 index 000000000..5e56bb192 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisIndexGenerator.java @@ -0,0 +1,46 @@ +package sqlancer.doris.gen; + +import java.sql.SQLException; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; + +public final class DorisIndexGenerator { + + private DorisIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) throws SQLException { + if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { + throw new IgnoreMeException(); + } + ExpectedErrors errors = new ExpectedErrors(); + + DorisTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); + String indexName = globalState.getSchema().getFreeIndexName(); + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("INDEX "); + if (Randomly.getBoolean()) { + sb.append("IF NOT EXISTS "); + } + sb.append(indexName); + sb.append(" ON "); + sb.append(randomTable.getName()); + sb.append("("); + int nr = 1; // Doris Only support CREATE_INDEX on single column and index type is BITMAP; + List subset = Randomly.extractNrRandomColumns(randomTable.getColumns(), nr); + sb.append(subset.get(0).getName()); + sb.append(") "); + if (Randomly.getBoolean()) { + sb.append("USING BITMAP "); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java new file mode 100644 index 000000000..7ce4d2e60 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -0,0 +1,56 @@ +package sqlancer.doris.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisInsertGenerator extends AbstractInsertGenerator { + + private final DorisGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public DorisInsertGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + return new DorisInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyInsertColumns(); + sb.append(table.getName()); + sb.append(" ("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + DorisErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertValue(DorisColumn column) { + if (column.hasDefaultValue() && Randomly.getBooleanWithRatherLowProbability()) { + sb.append("DEFAULT"); + } else { + String value = DorisToStringVisitor + .asString(DorisExprToNode.cast(new DorisNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); // 生成一个与column相同的常量类型 + sb.append(value); + } + } + +} diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java new file mode 100644 index 000000000..5dcb82e8f --- /dev/null +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -0,0 +1,427 @@ +package sqlancer.doris.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.DorisSchema.DorisRowValue; +import sqlancer.doris.ast.DorisAggregateOperation; +import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; +import sqlancer.doris.ast.DorisBetweenOperation; +import sqlancer.doris.ast.DorisBinaryArithmeticOperation; +import sqlancer.doris.ast.DorisBinaryArithmeticOperation.DorisBinaryArithmeticOperator; +import sqlancer.doris.ast.DorisBinaryComparisonOperation; +import sqlancer.doris.ast.DorisBinaryComparisonOperation.DorisBinaryComparisonOperator; +import sqlancer.doris.ast.DorisBinaryLogicalOperation; +import sqlancer.doris.ast.DorisBinaryLogicalOperation.DorisBinaryLogicalOperator; +import sqlancer.doris.ast.DorisCaseOperation; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunctionOperation.DorisFunction; +import sqlancer.doris.ast.DorisInOperation; +import sqlancer.doris.ast.DorisLikeOperation; +import sqlancer.doris.ast.DorisOrderByTerm; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; +import sqlancer.doris.visitor.DorisExprToNode; + +public class DorisNewExpressionGenerator extends TypedExpressionGenerator { + + private final DorisGlobalState globalState; + + private final int maxDepth; + private boolean allowAggregateFunctions; + private DorisRowValue rowValue; + + private Set columnOfLeafNode; + + public DorisNewExpressionGenerator setRowValue(DorisRowValue rowValue) { + this.rowValue = rowValue; + return this; + } + + public void setColumnOfLeafNode(Set columnOfLeafNode) { + this.columnOfLeafNode = columnOfLeafNode; + } + + public DorisNewExpressionGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + } + + @Override + public DorisExpression generateLeafNode(DorisDataType dataType) { + if (Randomly.getBoolean()) { + return generateConstant(dataType); + } else { + if (filterColumns(dataType).isEmpty()) { + return generateConstant(dataType); + } else { + return createColumnOfType(dataType); + } + } + } + + final List filterColumns(DorisDataType dataType) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType().getPrimitiveDataType() == dataType) + .collect(Collectors.toList()); + } + } + + private DorisExpression createColumnOfType(DorisDataType type) { + List columns = filterColumns(type); + DorisColumn column = Randomly.fromList(columns); + DorisConstant value = rowValue == null ? null : rowValue.getValues().get(column); + if (columnOfLeafNode != null) { + columnOfLeafNode.add(DorisColumnValue.create(column, value)); + } + return DorisColumnValue.create(column, value); + } + + public List> generateOrderBy() { + List randomColumns = Randomly.subset(columns); + return randomColumns.stream() + .map(c -> new DorisOrderByTerm(new DorisColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) + .collect(Collectors.toList()); + } + + @Override + public DorisExpression generateExpression(DorisDataType type, int depth) { + // todo: case operation should be add into generateExpression + + if (Randomly.getBooleanWithRatherLowProbability() || depth >= maxDepth) { + return generateLeafNode(type); + } + + if (globalState.getDbmsSpecificOptions().testFunctions && Randomly.getBooleanWithRatherLowProbability()) { + List applicableFunctions = DorisFunction.getFunctionsCompatibleWith(type); + if (!applicableFunctions.isEmpty()) { + DorisFunction function = Randomly.fromList(applicableFunctions); + return function.getCall(type, this, depth + 1); + } + } + if (globalState.getDbmsSpecificOptions().testCasts && Randomly.getBooleanWithRatherLowProbability()) { + return new DorisCastOperation(DorisExprToNode.cast(generateExpression(getRandomType(), depth + 1)), type); + } + if (globalState.getDbmsSpecificOptions().testCase && Randomly.getBooleanWithRatherLowProbability()) { + DorisExpression expr = generateExpression(DorisDataType.BOOLEAN, depth + 1); + List conditions = new ArrayList<>(); + List cases = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + conditions.add(generateExpression(DorisDataType.BOOLEAN, depth + 1)); + cases.add(generateExpression(type, depth + 1)); + } + DorisExpression elseExpr = null; + if (Randomly.getBoolean()) { + elseExpr = generateExpression(type, depth + 1); + } + return new DorisCaseOperation(expr, conditions, cases, elseExpr); + } + + switch (type) { + case INT: + return generateIntExpression(depth); + case BOOLEAN: + return generateBooleanExpression(depth); + case FLOAT: + case DECIMAL: + case DATE: + case DATETIME: + case VARCHAR: + case NULL: + return generateConstant(type); + default: + throw new AssertionError(); + } + } + + public List generateExpressions(int nr, DorisDataType type) { + List expressions = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + expressions.add(generateExpression(type)); + } + return expressions; + } + + private enum IntExpression { + UNARY_OPERATION, BINARY_ARITHMETIC_OPERATION + } + + private DorisExpression generateIntExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } + IntExpression intExpression = Randomly.fromOptions(IntExpression.values()); + switch (intExpression) { + case UNARY_OPERATION: + return new DorisUnaryPrefixOperation(generateExpression(DorisDataType.INT, depth + 1), + Randomly.getBoolean() ? DorisUnaryPrefixOperator.UNARY_PLUS : DorisUnaryPrefixOperator.UNARY_MINUS); + case BINARY_ARITHMETIC_OPERATION: + return new DorisBinaryArithmeticOperation(generateExpression(DorisDataType.INT, depth + 1), + generateExpression(DorisDataType.INT, depth + 1), + Randomly.fromOptions(DorisBinaryArithmeticOperator.values())); + default: + throw new AssertionError(); + } + } + + private enum BooleanExpression { + POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, LIKE, BETWEEN, IN_OPERATION; + // SIMILAR_TO, POSIX_REGEX, BINARY_RANGE_COMPARISON,FUNCTION, CAST,; + } + + DorisExpression generateBooleanExpression(int depth) { + if (allowAggregateFunctions) { + allowAggregateFunctions = false; + } + List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (!globalState.getDbmsSpecificOptions().testIn) { + validOptions.remove(BooleanExpression.IN_OPERATION); + } + if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { + validOptions.remove(BooleanExpression.BINARY_LOGICAL_OPERATOR); + } + if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { + validOptions.remove(BooleanExpression.BINARY_COMPARISON); + } + if (!globalState.getDbmsSpecificOptions().testBetween) { + validOptions.remove(BooleanExpression.BETWEEN); + } + + BooleanExpression option = Randomly.fromList(validOptions); + switch (option) { + case POSTFIX_OPERATOR: + return getPostfix(depth + 1); + case NOT: + return getNOT(depth + 1); + case BETWEEN: + return getBetween(depth + 1); + case IN_OPERATION: + return getIn(depth + 1); + case BINARY_LOGICAL_OPERATOR: + return getBinaryLogical(depth + 1, DorisDataType.BOOLEAN); + case BINARY_COMPARISON: + return getComparison(depth + 1); + case LIKE: + return getLike(depth + 1, DorisDataType.VARCHAR); + default: + throw new AssertionError(); + } + + } + + DorisExpression getPostfix(int depth) { + DorisUnaryPostfixOperator randomOp = DorisUnaryPostfixOperator.getRandom(); + return new DorisUnaryPostfixOperation( + generateExpression(Randomly.fromOptions(randomOp.getInputDataTypes()), depth), randomOp); + } + + DorisExpression getNOT(int depth) { + DorisUnaryPrefixOperator op = DorisUnaryPrefixOperator.NOT; + return new DorisUnaryPrefixOperation(generateExpression(op.getRandomInputDataTypes(), depth), op); + } + + DorisExpression getBetween(int depth) { + DorisDataType dataType = Randomly.fromList(Arrays.asList(DorisDataType.values()).stream() + .filter(t -> t != DorisDataType.BOOLEAN).collect(Collectors.toList())); + + return new DorisBetweenOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + generateExpression(dataType, depth), Randomly.getBoolean()); + } + + DorisExpression getIn(int depth) { + DorisDataType dataType = Randomly.fromOptions(DorisDataType.values()); + DorisExpression leftExpr = generateExpression(dataType, depth); + List rightExprs = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + rightExprs.add(generateExpression(dataType, depth)); + } + return new DorisInOperation(leftExpr, rightExprs, Randomly.getBoolean()); + } + + DorisExpression getBinaryLogical(int depth, DorisDataType dataType) { + DorisExpression expr = generateExpression(dataType, depth); + int nr = Randomly.smallNumber() + 1; + for (int i = 0; i < nr; i++) { + expr = new DorisBinaryLogicalOperation(expr, generateExpression(DorisDataType.BOOLEAN, depth), + DorisBinaryLogicalOperator.getRandom()); + } + return expr; + } + + DorisExpression getComparison(int depth) { + // 跳过boolean + DorisDataType dataType = Randomly.fromList(Arrays.asList(DorisDataType.values()).stream() + .filter(t -> t != DorisDataType.BOOLEAN).collect(Collectors.toList())); + DorisExpression leftExpr = generateExpression(dataType, depth); + DorisExpression rightExpr = generateExpression(dataType, depth); + return new DorisBinaryComparisonOperation(leftExpr, rightExpr, + Randomly.fromOptions(DorisBinaryComparisonOperator.values())); + } + + DorisExpression getLike(int depth, DorisDataType dataType) { + return new DorisLikeOperation(generateExpression(dataType, depth), generateExpression(dataType, depth), + DorisLikeOperation.DorisLikeOperator.LIKE_OPERATOR); + } + + public DorisExpression generateExpressionWithExpectedResult(DorisDataType type) { + DorisExpression expr; + do { + expr = this.generateExpression(type); + } while (expr.getExpectedValue() == null); + return expr; + } + + @Override + public DorisExpression generatePredicate() { + return generateExpression(DorisDataType.BOOLEAN); + } + + @Override + public DorisExpression negatePredicate(DorisExpression predicate) { + return new DorisUnaryPrefixOperation(predicate, DorisUnaryPrefixOperator.NOT); + } + + @Override + public DorisExpression isNull(DorisExpression predicate) { + return new DorisUnaryPostfixOperation(predicate, DorisUnaryPostfixOperator.IS_NULL); + } + + public DorisExpression generateConstant(DorisDataType type, boolean isNullable) { + if (isNullable && Randomly.getBooleanWithSmallProbability()) { + createConstant(DorisDataType.NULL); + } + return createConstant(type); + } + + @Override + public DorisExpression generateConstant(DorisDataType type) { + if (Randomly.getBooleanWithSmallProbability()) { + return DorisConstant.createNullConstant(); + } + return createConstant(type); + } + + public DorisExpression createConstant(DorisDataType type) { + Randomly r = globalState.getRandomly(); + long timestamp; + switch (type) { + case INT: + if (globalState.getDbmsSpecificOptions().testIntConstants) { + return DorisConstant.createIntConstant(r.getInteger()); + } + return DorisConstant.createNullConstant(); + case BOOLEAN: + if (globalState.getDbmsSpecificOptions().testBooleanConstants) { + return DorisConstant.createBooleanConstant(Randomly.getBoolean()); + } + return DorisConstant.createNullConstant(); + case DECIMAL: + if (globalState.getDbmsSpecificOptions().testDecimalConstants) { + double v = r.getDouble(); + while (v == Double.MAX_VALUE || v == -Double.MAX_VALUE || v == Double.POSITIVE_INFINITY + || v == Double.NEGATIVE_INFINITY) { + v = r.getDouble(); + } + + // e.g. format 1234.413232532 to qualify num 34.4132 + String formatter = "%." + type.getDecimalScale() + "f"; + String vStr = String.format(formatter, v); + int pointPos = vStr.indexOf('.'); + if (pointPos > type.getDecimalPrecision() - type.getDecimalScale()) { + vStr = vStr.substring(pointPos - (type.getDecimalPrecision() - type.getDecimalScale())); + } + return DorisConstant.createFloatConstant(Double.parseDouble(vStr)); + } + return DorisConstant.createNullConstant(); + case FLOAT: + if (globalState.getDbmsSpecificOptions().testFloatConstants) { + return DorisConstant.createFloatConstant((float) r.getDouble()); + } + return DorisConstant.createNullConstant(); + case DATE: + if (globalState.getDbmsSpecificOptions().testDateConstants) { + // [1970-01-01 08:00:00, 3000-01-01 00:00:00] + timestamp = globalState.getRandomly().getLong(0, 32503651200L); + return DorisConstant.createDateConstant(timestamp); + } + return DorisConstant.createNullConstant(); + case DATETIME: + if (globalState.getDbmsSpecificOptions().testDateTimeConstants) { + // [1970-01-01 08:00:00, 3000-01-01 00:00:00] + timestamp = globalState.getRandomly().getLong(0, 32503651200L); + return Randomly.fromOptions(DorisConstant.createDatetimeConstant(timestamp), + DorisConstant.createDatetimeConstant()); + } + return DorisConstant.createNullConstant(); + case VARCHAR: + if (globalState.getDbmsSpecificOptions().testStringConstants) { + String s = r.getString(); + if (s.length() > type.getVarcharLength()) { + s = s.substring(0, type.getVarcharLength()); + } + return DorisConstant.createStringConstant(s); + } + return DorisConstant.createNullConstant(); + case NULL: + return DorisConstant.createNullConstant(); + default: + throw new AssertionError(type); + } + } + + @Override + protected DorisExpression generateColumn(DorisDataType type) { + return null; + } + + @Override + protected DorisDataType getRandomType() { + return Randomly.fromOptions(DorisDataType.values()); + } + + @Override + protected boolean canGenerateColumnOfType(DorisDataType type) { + return false; + } + + public DorisExpression generateArgsForAggregate(DorisAggregateFunction aggregateFunction) { + DorisDataType dataType = Randomly.fromOptions(DorisDataType.values()); + return new DorisAggregateOperation(generateExpressions(aggregateFunction.getNrArgs(), dataType), + aggregateFunction); + } + + public DorisExpression generateAggregate() { + DorisAggregateFunction aggrFunc = DorisAggregateFunction.getRandom(); + return generateArgsForAggregate(aggrFunc); + } + + public DorisExpression generateHavingClause() { + allowAggregateFunctions = true; + DorisExpression expression = generateExpression(DorisDataType.BOOLEAN); + allowAggregateFunctions = false; + return expression; + } + + public void setAllowAggregateFunctions(boolean allowAggregateFunctions) { + this.allowAggregateFunctions = allowAggregateFunctions; + } +} diff --git a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java new file mode 100644 index 000000000..3193594df --- /dev/null +++ b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java @@ -0,0 +1,79 @@ +package sqlancer.doris.gen; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.visitor.DorisExprToNode; + +public final class DorisRandomQuerySynthesizer { + + private DorisRandomQuerySynthesizer() { + } + + public static DorisSelect generateSelect(DorisGlobalState globalState, int nrColumns) { + DorisTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + List targetColumns = targetTables.getColumns(); + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState).setColumns(targetColumns); + DorisSelect select = new DorisSelect(); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); + int freeColumns = targetColumns.size(); + select.setDistinct(DorisSelect.DorisSelectDistinctType.getRandomWithoutNull()); + List> columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + Node column = null; + if (freeColumns > 0 && Randomly.getBoolean()) { + column = new DorisColumnValue(targetColumns.get(freeColumns - 1), null); + freeColumns -= 1; + columnOfLeafNode.add((DorisColumnValue) column); + } else { + column = DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); + } + columns.add(column); + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DorisJoin.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + if (Randomly.getBoolean()) { + select.setHavingClause(DorisExprToNode.cast(gen.generateHavingClause())); + } + if (Randomly.getBoolean()) { + select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + } + + List> noExprColumns = new ArrayList<>(columnOfLeafNode); + + if (Randomly.getBoolean()) { + select.setOrderByExpressions(Randomly.nonEmptySubset(noExprColumns)); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(noExprColumns); + } + if (Randomly.getBoolean()) { + select.setLimitClause(DorisConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(DorisConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + return select; + } + +} diff --git a/src/sqlancer/doris/gen/DorisTableGenerator.java b/src/sqlancer/doris/gen/DorisTableGenerator.java new file mode 100644 index 000000000..8a98739f5 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisTableGenerator.java @@ -0,0 +1,114 @@ +package sqlancer.doris.gen; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisTableGenerator { + + // private final ExpectedErrors errors = new ExpectedErrors(); + + public static SQLQueryAdapter createRandomTableStatement(DorisGlobalState globalState) throws SQLException { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + return new DorisTableGenerator().getQuery(globalState); + } + + public SQLQueryAdapter getQuery(DorisGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + DorisSchema.DorisTableDataModel dataModel = DorisSchema.DorisTableDataModel.getRandom(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(globalState); + Collections.sort(columns); + if (columns.isEmpty() || !columns.get(0).isKey()) { + return null; // ensure table has at least one key column + } + sb.append(columns.stream().map(DorisColumn::toString).collect(Collectors.joining(", "))); + sb.append(")"); + + List keysColumn = columns.stream().filter(DorisColumn::isKey).collect(Collectors.toList()); + if (globalState.getDbmsSpecificOptions().testDataModel && Randomly.getBoolean() && !keysColumn.isEmpty()) { + sb.append(" " + dataModel).append(" KEY("); + sb.append(keysColumn.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + } + sb.append(generateDistributionStr(globalState, dataModel, keysColumn)); + sb.append(" PROPERTIES (\"replication_num\" = \"1\")"); // now only consider this one parameter + DorisErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + public static String generateDistributionStr(DorisGlobalState globalState, + DorisSchema.DorisTableDataModel dataModel, List keysColumn) { + // DISTRIBUTED BY HASH (k1[,k2 ...]) [BUCKETS num] + // DISTRIBUTED BY RANDOM [BUCKETS num] + StringBuilder sb = new StringBuilder(); + sb.append(" DISTRIBUTED BY"); + if (dataModel == DorisSchema.DorisTableDataModel.UNIQUE || Randomly.getBoolean()) { + sb.append(" HASH ("); + sb.append(Randomly.nonEmptySubset(keysColumn).stream().map(DorisColumn::getName) + .collect(Collectors.joining(", "))); + sb.append(")"); + } else { + sb.append(" RANDOM"); + } + if (Randomly.getBoolean()) { + sb.append(" BUCKETS ").append(globalState.getRandomly().getInteger(1, 32)); + } + return sb.toString(); + } + + private static List getNewColumns(DorisGlobalState globalState) { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + DorisCompositeDataType columnType = DorisCompositeDataType.getRandomWithoutNull(); + columnType.initColumnArgs(); // set decimalAndVarchar + + boolean iskey = columnType.canBeKey() && Randomly.getBoolean(); + boolean isNullable = Randomly.getBoolean(); + if (!globalState.getDbmsSpecificOptions().testNotNullConstraints) { + isNullable = true; + } + // boolean isHllOrBitmap = (columnType.getPrimitiveDataType() == DorisSchema.DorisDataType.HLL) + // || (columnType.getPrimitiveDataType() == DorisSchema.DorisDataType.BITMAP); + boolean isHllOrBitmap = false; + DorisSchema.DorisColumnAggrType aggrType = DorisSchema.DorisColumnAggrType.NULL; + if (globalState.getDbmsSpecificOptions().testColumnAggr && (isHllOrBitmap || !iskey)) { + aggrType = DorisSchema.DorisColumnAggrType.getRandom(columnType); + } + + boolean hasDefaultValue = globalState.getDbmsSpecificOptions().testDefaultValues && Randomly.getBoolean() + && !isHllOrBitmap; + String defaultValue = ""; + if (hasDefaultValue) { + defaultValue = DorisToStringVisitor + .asString(DorisExprToNode.cast(new DorisNewExpressionGenerator(globalState) + .generateConstant(columnType.getPrimitiveDataType(), isNullable))); + } + columns.add(new DorisColumn(columnName, columnType, iskey, isNullable, aggrType, hasDefaultValue, + defaultValue)); + } + return columns; + } + +} diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java new file mode 100644 index 000000000..74fb9383d --- /dev/null +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -0,0 +1,56 @@ +package sqlancer.doris.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public final class DorisUpdateGenerator extends AbstractUpdateGenerator { + + private final DorisGlobalState globalState; + private DorisNewExpressionGenerator gen; + + private DorisUpdateGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + return new DorisUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new DorisNewExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + sb.append(" WHERE "); + sb.append(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + DorisErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void updateValue(DorisColumn column) { + if (Randomly.getBooleanWithSmallProbability()) { + DorisExpression expr = gen.generateExpression(column.getType().getPrimitiveDataType()); + sb.append(DorisToStringVisitor.asString(DorisExprToNode.cast(expr))); + } else { + DorisExpression expr = gen.generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()); + sb.append(DorisToStringVisitor.asString(expr)); + } + + } + +} diff --git a/src/sqlancer/doris/gen/DorisViewGenerator.java b/src/sqlancer/doris/gen/DorisViewGenerator.java new file mode 100644 index 000000000..480607378 --- /dev/null +++ b/src/sqlancer/doris/gen/DorisViewGenerator.java @@ -0,0 +1,38 @@ +package sqlancer.doris.gen; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public final class DorisViewGenerator { + + private DorisViewGenerator() { + } + + public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { + if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { + throw new IgnoreMeException(); + } + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(DorisToStringVisitor.asString(DorisRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + DorisErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + +} diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java new file mode 100644 index 000000000..dab6b53ae --- /dev/null +++ b/src/sqlancer/doris/oracle/DorisNoRECOracle.java @@ -0,0 +1,137 @@ +package sqlancer.doris.oracle; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisNoRECOracle extends NoRECBase implements TestOracle { + + private final DorisSchema s; + + public DorisNoRECOracle(DorisGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + DorisTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state).setColumns(columns); + Node randomWhereCondition = DorisExprToNode + .cast(gen.generateExpression(DorisDataType.BOOLEAN)); + List tables = randomTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DorisJoin.getJoins(tableList, state); + int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), + randomWhereCondition, joins); + int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, + randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getUnoptimizedQueryCount(List> tableList, + Node randomWhereCondition, List> joins) throws SQLException { + DorisSelect select = new DorisSelect(); + Node asText = new NewPostfixTextNode<>(new DorisCastOperation( + new NewPostfixTextNode(randomWhereCondition, + " IS NOT NULL AND " + DorisToStringVisitor.asString(randomWhereCondition)), + new DorisCompositeDataType(DorisDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + select.setFromList(tableList); + select.setJoinList(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + DorisToStringVisitor.asString(select) + ") as res"; + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getOptimizedQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, List> joins) + throws SQLException { + DorisSelect select = new DorisSelect(); + // select.setGroupByClause(groupBys); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + List> constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByExpressions(constants); + } + select.setJoinList(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = DorisToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java new file mode 100644 index 000000000..95c79393b --- /dev/null +++ b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java @@ -0,0 +1,155 @@ +package sqlancer.doris.oracle; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.PivotedQuerySynthesisBase; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.DorisSchema.DorisRowValue; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.visitor.DorisExpectedValueVisitor; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisPivotedQuerySynthesisOracle + extends PivotedQuerySynthesisBase { + + private List fetchColumns; + + public DorisPivotedQuerySynthesisOracle(DorisGlobalState globalState) { + super(globalState); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + protected Query getRectifiedQuery() throws Exception { + DorisTables randomTables = globalState.getSchema().getRandomTableNonEmptyAndViewTables(); + List columns = randomTables.getColumns(); + DorisSelect selectStatement = new DorisSelect(); + boolean isDistinct = Randomly.getBoolean(); + selectStatement.setDistinct(isDistinct); + pivotRow = randomTables.getRandomRowValue(globalState.getConnection()); + fetchColumns = columns; + selectStatement.setFetchColumns(fetchColumns.stream() + .map(c -> new DorisColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) + .collect(Collectors.toList())); + selectStatement.setFromList(randomTables.getTables().stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList())); + DorisExpression whereClause = generateRectifiedExpression(columns, pivotRow); + selectStatement.setWhereClause(DorisExprToNode.cast(whereClause)); + List> groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setGroupByExpressions(groupByClause); + Node limitClause = generateLimit(); + selectStatement.setLimitClause(limitClause); + if (limitClause != null) { + Node offsetClause = generateOffset(); + selectStatement.setOffsetClause(offsetClause); + } + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + gen.setColumns(columns); + if (!isDistinct) { + List> constants = new ArrayList<>(); + constants.add(new DorisConstant.DorisIntConstant( + Randomly.smallNumber() % selectStatement.getFetchColumns().size() + 1)); + selectStatement.setOrderByExpressions(constants); + } + return new SQLQueryAdapter(DorisToStringVisitor.asString(selectStatement), errors); + } + + private DorisExpression generateRectifiedExpression(List columns, DorisRowValue pivotRow) { + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState).setColumns(columns); + gen.setRowValue(pivotRow); + DorisExpression expr = gen.generateExpressionWithExpectedResult(DorisDataType.BOOLEAN); + DorisExpression result = null; + if (expr.getExpectedValue().isNull()) { + result = new DorisUnaryPostfixOperation(expr, DorisUnaryPostfixOperation.DorisUnaryPostfixOperator.IS_NULL); + } else if (!expr.getExpectedValue().cast(DorisDataType.BOOLEAN).asBoolean()) { + result = new DorisUnaryPrefixOperation(expr, DorisUnaryPrefixOperation.DorisUnaryPrefixOperator.NOT); + } + rectifiedPredicates.add(result); + return result; + } + + @Override + protected Query getContainmentCheckQuery(Query pivotRowQuery) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("SELECT * FROM ("); + sb.append(pivotRowQuery.getUnterminatedQueryString()); + sb.append(") as result WHERE "); + int i = 0; + for (DorisColumn c : fetchColumns) { + if (i++ != 0) { + sb.append(" AND "); + } + sb.append("result."); + sb.append(c.getTable().getName()); + sb.append(c.getName()); + if (pivotRow.getValues().get(c).isNull()) { + sb.append(" IS NULL "); + } else { + sb.append(" = "); + sb.append(pivotRow.getValues().get(c).toString()); + } + } + String resultingQueryString = sb.toString(); + return new SQLQueryAdapter(resultingQueryString, errors); + } + + private DorisColumn getFetchValueAliasedColumn(DorisColumn c) { + DorisColumn aliasedColumn = new DorisColumn(c.getName() + " AS " + c.getTable().getName() + c.getName(), + c.getType(), false, false); + aliasedColumn.setTable(c.getTable()); + return aliasedColumn; + } + + @Override + protected String getExpectedValues(DorisExpression expr) { + return DorisExpectedValueVisitor.asExpectedValues(DorisExprToNode.cast(expr)); + } + + private List> generateGroupByClause(List columns, DorisRowValue rowValue) { + if (Randomly.getBoolean()) { + return columns.stream().map(c -> new DorisColumnValue(c, rowValue.getValues().get(c))) + .collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + private Node generateLimit() { + if (Randomly.getBoolean()) { + return DorisConstant.createIntConstant(Integer.MAX_VALUE); + } else { + return null; + } + } + + private Node generateOffset() { + if (Randomly.getBoolean()) { + return DorisConstant.createIntConstant(0); + } else { + return null; + } + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java new file mode 100644 index 000000000..2e886afbe --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -0,0 +1,194 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; +import sqlancer.doris.DorisSchema.DorisDataType; +import sqlancer.doris.ast.DorisAggregateOperation; +import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; +import sqlancer.doris.ast.DorisBinaryArithmeticOperation; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; +import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningAggregateTester extends DorisQueryPartitioningBase + implements TestOracle { + + private String firstResult; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public DorisQueryPartitioningAggregateTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + DorisAggregateFunction aggregateFunction = Randomly.fromOptions(DorisAggregateFunction.MAX, + DorisAggregateFunction.MIN, DorisAggregateFunction.SUM, DorisAggregateFunction.COUNT, + DorisAggregateFunction.AVG); + NewFunctionNode aggregate = (DorisAggregateOperation) gen + .generateArgsForAggregate(aggregateFunction); + List> fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + while (Randomly.getBooleanWithRatherLowProbability()) { + fetchColumns.add((DorisAggregateOperation) gen.generateAggregate()); + } + select.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List> constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByExpressions(constants); + } + originalQuery = DorisToStringVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResult == null && secondResult != null + || firstResult != null && (!firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { + throw new AssertionError(); + } + + } + + private String createMetamorphicUnionQuery(DorisSelect select, + NewFunctionNode aggregate, List> from) { + String metamorphicQuery; + Node whereClause = DorisExprToNode.cast(gen.generateExpression(DorisDataType.BOOLEAN)); + Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + DorisUnaryPrefixOperator.NOT); + Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + DorisUnaryPostfixOperator.IS_NULL); + List> mappedAggregate = mapped(aggregate); + DorisSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + DorisSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + DorisSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += DorisToStringVisitor.asString(leftSelect) + " UNION ALL " + + DorisToStringVisitor.asString(middleSelect) + " UNION ALL " + + DorisToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List> mapped(NewFunctionNode aggregate) { + + DorisCastOperation count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(Arrays.asList(aggregate)); + case AVG: + NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + DorisAggregateFunction.SUM); + count = new DorisCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), + new DorisCompositeDataType(DorisDataType.FLOAT, 8)); + return aliasArgs(Arrays.asList(sum, count)); + case STDDEV_POP: + NewFunctionNode sumSquared = new NewFunctionNode<>( + Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DorisBinaryArithmeticOperation.DorisBinaryArithmeticOperator.MULTIPLICATION)), + DorisAggregateFunction.SUM); + count = new DorisCastOperation( + new NewFunctionNode(aggregate.getArgs(), + DorisAggregateFunction.COUNT), + new DorisCompositeDataType(DorisDataType.FLOAT, 8)); + NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), + DorisAggregateFunction.AVG); + return aliasArgs(Arrays.asList(sumSquared, count, avg)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List> aliasArgs(List> originalAggregateArgs) { + List> args = new ArrayList<>(); + int i = 0; + for (Node expr : originalAggregateArgs) { + args.add(new NewAliasNode(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(NewFunctionNode aggregate) { + switch (aggregate.getFunc()) { + case STDDEV_POP: + return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; + case AVG: + return "SUM(agg0::FLOAT)/SUM(agg1)::FLOAT"; + case COUNT: + return DorisAggregateFunction.SUM.toString() + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private DorisSelect getSelect(List> aggregates, List> from, + Node whereClause, List> joinList) { + DorisSelect leftSelect = new DorisSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(groupByExpression); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java new file mode 100644 index 000000000..5db230854 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java @@ -0,0 +1,98 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnValue; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.gen.DorisNewExpressionGenerator; + +public class DorisQueryPartitioningBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + DorisSchema s; + DorisTables targetTables; + DorisNewExpressionGenerator gen; + DorisSelect select; + + List> groupByExpression; + + public DorisQueryPartitioningBase(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + public static String canonicalizeResultValue(String value) { + if (value == null) { + return value; + } + + switch (value) { + case "-0.0": + return "0.0"; + case "-0": + return "0"; + default: + } + + return value; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new DorisNewExpressionGenerator(state).setColumns(targetTables.getColumns()); + HashSet columnOfLeafNode = new HashSet<>(); + gen.setColumnOfLeafNode(columnOfLeafNode); + initializeTernaryPredicateVariants(); + select = new DorisSelect(); + columnOfLeafNode.addAll(targetTables.getColumns().stream().map(c -> new DorisColumnValue(c, null)) + .collect(Collectors.toList())); + groupByExpression = new ArrayList<>(columnOfLeafNode); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = DorisJoin.getJoins(tableList, state); + select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new DorisColumn("*", null, false, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..6cc15fdc8 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java @@ -0,0 +1,45 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningDistinctTester extends DorisQueryPartitioningBase { + + public DorisQueryPartitioningDistinctTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(DorisExprToNode.cast(predicate)); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + + String unionString = "SELECT DISTINCT * FROM (" + firstQueryString + " UNION ALL " + secondQueryString + + " UNION ALL " + thirdQueryString + ") tmpTable"; + combinedString.add(unionString); + List secondResultSet = ComparatorHelper.getResultSetFirstColumnAsString(unionString, errors, state); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DorisQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..0761fddb6 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java @@ -0,0 +1,55 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningGroupByTester extends DorisQueryPartitioningBase { + + public DorisQueryPartitioningGroupByTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setGroupByExpressions(groupByExpression); + select.setWhereClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(DorisExprToNode.cast(predicate)); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DorisQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + List> generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java new file mode 100644 index 000000000..7ce10dc09 --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -0,0 +1,73 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningHavingTester extends DorisQueryPartitioningBase + implements TestOracle { + + public DorisQueryPartitioningHavingTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBoolean()) { + select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + } + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + List> constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByExpressions(constants); + } + select.setGroupByExpressions(groupByExpression); + select.setHavingClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setHavingClause(DorisExprToNode.cast(predicate)); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setHavingClause(DorisExprToNode.cast(negatedPredicate)); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setHavingClause(DorisExprToNode.cast(isNullPredicate)); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DorisQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + protected DorisExpression generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List> generateFetchColumns() { + gen.setAllowAggregateFunctions(true); + List> expressions = DorisExprToNode + .casts(gen.generateExpressions(Randomly.smallNumber() + 1)); + gen.setAllowAggregateFunctions(false); + return expressions; + } + +} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java new file mode 100644 index 000000000..c45bc36fb --- /dev/null +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -0,0 +1,53 @@ +package sqlancer.doris.oracle.tlp; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.DorisErrors; +import sqlancer.doris.DorisProvider.DorisGlobalState; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.visitor.DorisToStringVisitor; + +public class DorisQueryPartitioningWhereTester extends DorisQueryPartitioningBase { + + public DorisQueryPartitioningWhereTester(DorisGlobalState state) { + super(state); + DorisErrors.addExpressionErrors(errors); + DorisErrors.addInsertErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = DorisToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + List> constants = new ArrayList<>(); + constants.add( + new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByExpressions(constants); + } + select.setWhereClause(DorisExprToNode.cast(predicate)); + String firstQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + String secondQueryString = DorisToStringVisitor.asString(select); + select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + String thirdQueryString = DorisToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, DorisQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/doris/utils/DorisNumberUtils.java b/src/sqlancer/doris/utils/DorisNumberUtils.java new file mode 100644 index 000000000..01dd7238d --- /dev/null +++ b/src/sqlancer/doris/utils/DorisNumberUtils.java @@ -0,0 +1,111 @@ +package sqlancer.doris.utils; + +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public final class DorisNumberUtils { + private static Pattern numberPattern = Pattern.compile("-?[0-9]+(\\\\.[0-9]+)?"); + private static Pattern integerPattern = Pattern.compile("^[-\\+]?[\\d]*$"); + private static Pattern datePattern = Pattern + .compile("^([1-9]\\d{3}-)(([0]{0,1}[1-9]-)|([1][0-2]-))(([0-3]{0,1}[0-9]))$"); + private static Pattern datetimePattern = Pattern.compile( + "((([0-9]{3}[1-9]|[0-9]{2}[1-9][0-9]{1}|[0-9]{1}[1-9][0-9]{2}|[1-9][0-9]{3})-(((0[13578]|1[02])-(0[1-9]|[12][0-9]|3[01]))|((0[469]|11)-(0[1-9]|[12][0-9]|30))|(02-(0[1-9]|[1][0-9]|2[0-8]))))|((([0-9]{2})(0[48]|[2468][048]|[13579][26])|((0[48]|[2468][048]|[3579][26])00))-02-29))\\\\s+([0-1]?[0-9]|2[0-3]):([0-5][0-9]):([0-5][0-9])\n"); + + private DorisNumberUtils() { + } + + public static boolean isNumber(String str) { + Matcher m = numberPattern.matcher(str); + return m.matches(); + } + + public static boolean isInteger(String str) { + Matcher m = integerPattern.matcher(str); + return m.matches(); + } + + public static boolean isDate(String str) { + Matcher m = datePattern.matcher(str); + return m.matches(); + } + + public static boolean isDatetime(String str) { + Matcher m = datetimePattern.matcher(str); + return m.matches(); + } + + public static String timestampToDateText(long ts) { + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + return dateFormat.format(ts); + } + + public static String timestampToDatetimeText(long ts) { + SimpleDateFormat datetimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return datetimeFormat.format(ts); + } + + public static String dateTextToDatetimeText(String date) { + // '2021-03-12' -> '2021-03-12 00:00:00' + return date + " 00:00:00"; + } + + public static String datetimeTextToDateText(String datetime) { + // '2021-03-12 00:00:00' -> '2021-03-12' + return datetime.substring(0, 10); + } + + public static boolean datetimeEqual(String dt1, String dt2) { + String datetime1 = dt1; + String datetime2 = dt2; + if (isDate(dt1)) { + datetime1 = dateTextToDatetimeText(dt1); + } + if (isDate(dt2)) { + datetime2 = dateTextToDatetimeText(dt2); + } + return datetime1.contentEquals(datetime2); + } + + public static boolean dateEqual(String d1, String d2) { + String date1 = d1; + String date2 = d2; + if (isDatetime(d1)) { + date1 = datetimeTextToDateText(d1); + } + if (isDatetime(d2)) { + date2 = datetimeTextToDateText(d2); + } + return date1.contentEquals(date2); + } + + public static boolean dateLessThan(String d1, String d2) { + String date1 = d1; + String date2 = d2; + if (isDatetime(d1)) { + date1 = datetimeTextToDateText(d1); + } + if (isDatetime(d2)) { + date2 = datetimeTextToDateText(d2); + } + return date1.compareTo(date2) < 0; + } + + public static boolean datetimeLessThan(String dt1, String dt2) { + String datetime1 = dt1; + String datetime2 = dt2; + if (isDate(dt1)) { + datetime1 = dateTextToDatetimeText(dt1); + } + if (isDate(dt2)) { + datetime2 = dateTextToDatetimeText(dt2); + } + return datetime1.compareTo(datetime2) < 0; + } + + public static String getCurrentTimeText() { + SimpleDateFormat datetimeFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return datetimeFormat.format(new Date()); + } +} diff --git a/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java new file mode 100644 index 000000000..57cb29ae0 --- /dev/null +++ b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java @@ -0,0 +1,155 @@ +package sqlancer.doris.visitor; + +import java.util.List; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; + +public class DorisExpectedValueVisitor { + + protected final StringBuilder sb = new StringBuilder(); + + private void print(Node expr) { + sb.append(DorisToStringVisitor.asString(expr)); + sb.append(" -- "); + sb.append(((DorisExpression) expr).getExpectedValue()); + sb.append("\n"); + } + + @SuppressWarnings("unchecked") + public void visit(Node expr) { + assert expr != null; + if (expr instanceof ColumnReferenceNode) { + visit((ColumnReferenceNode) expr); + } else if (expr instanceof NewUnaryPostfixOperatorNode) { + visit((NewUnaryPostfixOperatorNode) expr); + } else if (expr instanceof NewUnaryPrefixOperatorNode) { + visit((NewUnaryPrefixOperatorNode) expr); + } else if (expr instanceof NewBinaryOperatorNode) { + visit((NewBinaryOperatorNode) expr); + } else if (expr instanceof TableReferenceNode) { + visit((TableReferenceNode) expr); + } else if (expr instanceof NewFunctionNode) { + visit((NewFunctionNode) expr); + } else if (expr instanceof NewBetweenOperatorNode) { + visit((NewBetweenOperatorNode) expr); + } else if (expr instanceof NewInOperatorNode) { + visit((NewInOperatorNode) expr); + } else if (expr instanceof NewOrderingTerm) { + visit((NewOrderingTerm) expr); + } else if (expr instanceof NewAliasNode) { + visit((NewAliasNode) expr); + } else if (expr instanceof NewPostfixTextNode) { + visit((NewPostfixTextNode) expr); + } else if (expr instanceof DorisConstant) { + visit((DorisConstant) expr); + } else if (expr instanceof DorisSelect) { + visit((DorisSelect) expr); + } else if (expr instanceof DorisJoin) { + visit((DorisJoin) expr); + } else { + throw new AssertionError(expr); + } + } + + public void visit(ColumnReferenceNode c) { + print(c); + } + + public void visit(NewUnaryPostfixOperatorNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewUnaryPrefixOperatorNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewBinaryOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(TableReferenceNode t) { + print(t); + } + + public void visit(NewFunctionNode fun) { + print(fun); + visit(fun.getArgs()); + } + + public void visit(List> expressions) { + for (Node expression : expressions) { + visit(expression); + } + } + + public void visit(NewBetweenOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getMiddle()); + visit(op.getRight()); + } + + public void visit(NewInOperatorNode op) { + print(op); + visit(op.getLeft()); + visit(op.getRight()); + } + + public void visit(NewOrderingTerm op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewAliasNode op) { + print(op); + visit(op.getExpr()); + } + + public void visit(NewPostfixTextNode postFixText) { + print(postFixText); + visit(postFixText.getExpr()); + } + + public void visit(DorisConstant constant) { + print(constant); + } + + public void visit(DorisSelect select) { + print(select.getWhereClause()); + } + + public void visit(DorisJoin join) { + print(join.getOnCondition()); + } + + public String get() { + return sb.toString(); + } + + public static String asExpectedValues(Node expr) { + DorisExpectedValueVisitor v = new DorisExpectedValueVisitor(); + v.visit(expr); + return v.get(); + } + +} diff --git a/src/sqlancer/doris/visitor/DorisExprToNode.java b/src/sqlancer/doris/visitor/DorisExprToNode.java new file mode 100644 index 000000000..cec7aa061 --- /dev/null +++ b/src/sqlancer/doris/visitor/DorisExprToNode.java @@ -0,0 +1,25 @@ +package sqlancer.doris.visitor; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.ast.DorisExpression; + +public final class DorisExprToNode { + + private DorisExprToNode() { + + } + + @SuppressWarnings("unchecked") + public static Node cast(DorisExpression expression) { + return (Node) expression; + } + + @SuppressWarnings("unchecked") + public static List> casts(List expressions) { + return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/doris/visitor/DorisToStringVisitor.java b/src/sqlancer/doris/visitor/DorisToStringVisitor.java new file mode 100644 index 000000000..846955c83 --- /dev/null +++ b/src/sqlancer/doris/visitor/DorisToStringVisitor.java @@ -0,0 +1,175 @@ +package sqlancer.doris.visitor; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.doris.ast.DorisCaseOperation; +import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunctionOperation; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; + +public class DorisToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof DorisConstant) { + visit((DorisConstant) expr); + } else if (expr instanceof DorisSelect) { + visit((DorisSelect) expr); + } else if (expr instanceof DorisJoin) { + visit((DorisJoin) expr); + } else if (expr instanceof DorisCastOperation) { + visit((DorisCastOperation) expr); + } else if (expr instanceof DorisCaseOperation) { + visit((DorisCaseOperation) expr); + } else if (expr instanceof DorisFunctionOperation) { + visit((DorisFunctionOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(DorisJoin join) { + sb.append(" "); + visit(join.getLeftTable()); + sb.append(" "); + switch (join.getJoinType()) { + case INNER: + if (Randomly.getBoolean()) { + sb.append("INNER "); + } else { + sb.append("CROSS "); + } + sb.append("JOIN "); + break; + case LEFT: + sb.append("LEFT "); + if (Randomly.getBoolean()) { + sb.append(" OUTER "); + } + sb.append("JOIN "); + break; + case RIGHT: + sb.append("RIGHT "); + if (Randomly.getBoolean()) { + sb.append(" OUTER "); + } + sb.append("JOIN "); + break; + case STRAIGHT: + sb.append("STRAIGHT_JOIN "); + break; + default: + throw new AssertionError(); + } + visit(join.getRightTable()); + sb.append(" "); + if (join.getOnCondition() != null) { + sb.append("ON "); + visit(join.getOnCondition()); + } + } + + private void visit(DorisConstant constant) { + sb.append(constant.toString()); + } + + private void visit(DorisCastOperation castExpr) { + sb.append("CAST("); + visit(castExpr.getExpr()); + sb.append(" AS "); + sb.append(castExpr.getType().toString()); + sb.append(") "); + } + + private void visit(DorisFunctionOperation func) { + sb.append(func.getFunction().getFunctionName()); + sb.append("("); + + if (func.getArgs() != null) { + for (int i = 0; i < func.getArgs().size(); i++) { + visit(DorisExprToNode.cast(func.getArgs().get(i))); + if (i != func.getArgs().size() - 1) { + sb.append(","); + } + } + } + sb.append(") "); + } + + private void visit(DorisCaseOperation cases) { + sb.append("CASE "); + visit(DorisExprToNode.cast(cases.getExpr())); + sb.append(" "); + for (int i = 0; i < cases.getConditions().size(); i++) { + DorisExpression predicate = cases.getConditions().get(i); + DorisExpression then = cases.getThenClauses().get(i); + sb.append(" WHEN "); + visit(DorisExprToNode.cast(predicate)); + sb.append(" THEN "); + visit(DorisExprToNode.cast(then)); + sb.append(" "); + } + if (cases.getElseClause() != null) { + sb.append("ELSE "); + visit(DorisExprToNode.cast(cases.getElseClause())); + sb.append(" "); + } + sb.append("END "); + } + + private void visit(DorisSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public static String asString(Node expr) { + DorisToStringVisitor visitor = new DorisToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + public static String asString(DorisExpression expr) { + DorisToStringVisitor visitor = new DorisToStringVisitor(); + visitor.visit(DorisExprToNode.cast(expr)); + return visitor.get(); + } +} diff --git a/test/sqlancer/dbms/TestDoris.java b/test/sqlancer/dbms/TestDoris.java new file mode 100644 index 000000000..fd3c1e95a --- /dev/null +++ b/test/sqlancer/dbms/TestDoris.java @@ -0,0 +1,54 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDoris { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisNoREC() { + String dorisAvailable = System.getenv("DORIS_AVAILABLE"); + boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); + assumeTrue(dorisIsAvailable); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "NOREC")); + } + + @Test + public void testdorisPQS() { + String dorisAvailable = System.getenv("DORIS_AVAILABLE"); + boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); + assumeTrue(dorisIsAvailable); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "PQS")); + } + + @Test + public void testdorisTLPQueryPartitioning() { + String dorisAvailable = System.getenv("DORIS_AVAILABLE"); + boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); + assumeTrue(dorisIsAvailable); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "QUERY_PARTITIONING")); + } +} From d6263cd2cd3904854d24b12bfffcd0d7a4d086d4 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 22 May 2023 17:07:41 +0800 Subject: [PATCH 343/856] chore: upgrade versions in CI (#787) * chore: upgrade versions in CI * fix: typo and version * fix: add integers in mysql * fix: fix mysql Unknown system variable 'myisam_repair_threads' * fix: Set up typo --- .github/workflows/main.yml | 105 ++++++++---------- src/sqlancer/Randomly.java | 33 ++++++ src/sqlancer/mysql/gen/MySQLSetGenerator.java | 5 +- .../mysql/gen/MySQLTableGenerator.java | 8 +- 4 files changed, 88 insertions(+), 63 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c03e3908c..945d1bc18 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,18 +14,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - - name: Setup Python + - name: Set up Python uses: actions/setup-python@v4 with: python-version: '3.10' @@ -36,11 +36,11 @@ jobs: name: DBMS Tests (Citus) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -82,11 +82,11 @@ jobs: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -105,17 +105,15 @@ jobs: docker stop clickhouse-server docker rm clickhouse-server - - cockroachdb: name: DBMS Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -134,11 +132,11 @@ jobs: name: QPG Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -166,11 +164,11 @@ jobs: - 8000:8000 - 3307:3307 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -179,17 +177,16 @@ jobs: run: | DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test - duckdb: name: DBMS Tests (DuckDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build @@ -201,11 +198,11 @@ jobs: name: DBMS Tests (H2) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -215,14 +212,14 @@ jobs: mariadb: name: DBMS Tests (MariaDB) - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -243,7 +240,7 @@ jobs: name: DBMS Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Materialize @@ -254,7 +251,7 @@ jobs: # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -268,7 +265,7 @@ jobs: name: QPG Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up Materialize @@ -279,7 +276,7 @@ jobs: # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -291,25 +288,20 @@ jobs: mysql: name: DBMS Tests (MySQL) - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up MySQL run: | - sudo apt-get install libssl-dev libmecab2 libjson-perl mecab-ipadic-utf8 - sudo apt-get remove mysql-* - wget -q https://dev.mysql.com/get/Downloads/MySQL-8.0/mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar - tar -xvf mysql-server_8.0.20-1ubuntu18.04_amd64.deb-bundle.tar - sudo dpkg -i *.deb - sudo systemctl start mysql + sudo systemctl start mysql.service # MySQL 8.0.32-0ubuntu0.22.04.2 - name: Create SQLancer user run: mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" - name: Run Tests @@ -317,16 +309,15 @@ jobs: MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP - postgres: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up PostgreSQL - uses: harmon758/postgresql-action@v1 + uses: harmon758/postgresql-action@v1.0.0 with: postgresql version: '12' postgresql user: 'sqlancer' @@ -346,11 +337,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build @@ -360,17 +351,16 @@ jobs: mvn -Dtest=TestSQLitePQS test mvn -Dtest=TestSQLite3 test - sqlite-qpg: name: QPG Tests (SQLite) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build @@ -379,16 +369,15 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test - tidb: name: DBMS Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -407,11 +396,11 @@ jobs: name: QPG Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer @@ -430,16 +419,16 @@ jobs: name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 11 - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Setup Yugabyte + - name: Set up Yugabyte run: | docker pull yugabytedb/yugabyte:latest docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false @@ -493,11 +482,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 13 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 13 - name: Build @@ -510,11 +499,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 14 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 14 - name: Build @@ -527,11 +516,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 with: fetch-depth: 0 - name: Set up JDK 15 - uses: actions/setup-java@v1 + uses: actions/setup-java@v1.4.4 with: java-version: 15-ea - name: Build diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index ae34ba1b1..e1250abb3 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -16,6 +16,7 @@ public final class Randomly { private static int cacheSize = 100; private final List cachedLongs = new ArrayList<>(); + private final List cachedIntegers = new ArrayList<>(); private final List cachedStrings = new ArrayList<>(); private final List cachedDoubles = new ArrayList<>(); private final List cachedBytes = new ArrayList<>(); @@ -30,6 +31,12 @@ private void addToCache(long val) { } } + private void addToCache(int val) { + if (useCaching && cachedIntegers.size() < cacheSize && !cachedIntegers.contains(val)) { + cachedIntegers.add(val); + } + } + private void addToCache(double val) { if (useCaching && cachedDoubles.size() < cacheSize && !cachedDoubles.contains(val)) { cachedDoubles.add(val); @@ -50,6 +57,14 @@ private Long getFromLongCache() { } } + private Integer getFromIntegerCache() { + if (!useCaching || cachedIntegers.isEmpty()) { + return null; + } else { + return Randomly.fromList(cachedIntegers); + } + } + private Double getFromDoubleCache() { if (!useCaching) { return null; @@ -368,6 +383,24 @@ public long getPositiveInteger() { return value; } + public int getPositiveIntegerInt() { + if (cacheProbability()) { + Integer value = getFromIntegerCache(); + if (value != null && value >= 0) { + return value; + } + } + int value; + if (smallBiasProbability()) { + value = Randomly.fromOptions(0, Integer.MAX_VALUE, 1); + } else { + value = getNextInt(0, Integer.MAX_VALUE); + } + addToCache(value); + assert value >= 0; + return value; + } + public double getFiniteDouble() { while (true) { double val = getDouble(); diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 2db20fee9..00a477714 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -66,8 +66,9 @@ private enum Action { MAX_SP_RECURSION_DEPTH("max_sp_recursion_depth", (r) -> r.getLong(0, 255), Scope.GLOBAL, Scope.SESSION), // MYISAM_DATA_POINTER_SIZE("myisam_data_pointer_size", (r) -> r.getLong(2, 7), Scope.GLOBAL), // MYISAM_MAX_SORT_FILE_SIZE("myisam_max_sort_file_size", (r) -> r.getLong(0, 9223372036854775807L), Scope.GLOBAL), // - MYISAM_REPAIR_THREADS("myisam_repair_threads", (r) -> r.getLong(1, Long.MAX_VALUE), Scope.GLOBAL, - Scope.SESSION), // + // MYISAM_REPAIR_THREADS("myisam_repair_threads", (r) -> r.getLong(1, Long.MAX_VALUE), Scope.GLOBAL, + // Scope.SESSION), // comment out this to avoid java.sql.SQLException: Unknown system variable + // 'myisam_repair_threads' MYISAM_SORT_BUFFER_SIZE("myisam_sort_buffer_size", (r) -> r.getLong(4096, Long.MAX_VALUE), Scope.GLOBAL, Scope.SESSION), // MYISAM_STATS_METHOD("myisam_stats_method", diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 579d9a4c9..427f5b7c3 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -176,9 +176,10 @@ private void appendTableOptions() { sb.append("AUTO_INCREMENT = "); sb.append(r.getPositiveInteger()); break; + // The valid range for avg_row_length is [0,4294967295] case AVG_ROW_LENGTH: sb.append("AVG_ROW_LENGTH = "); - sb.append(r.getPositiveInteger()); + sb.append(r.getLong(0, 4294967295L + 1)); break; case CHECKSUM: sb.append("CHECKSUM = 1"); @@ -212,9 +213,10 @@ private void appendTableOptions() { sb.append("INSERT_METHOD = "); sb.append(Randomly.fromOptions("NO", "FIRST", "LAST")); break; + // The valid range for key_block_size is [0,65535] case KEY_BLOCK_SIZE: sb.append("KEY_BLOCK_SIZE = "); - sb.append(r.getPositiveInteger()); + sb.append(r.getInteger(0, 65535 + 1)); break; case MAX_ROWS: sb.append("MAX_ROWS = "); @@ -336,7 +338,7 @@ private void appendTypeString(MySQLDataType randomType) { if (Randomly.getBoolean()) { sb.append("("); sb.append(Randomly.getNotCachedInteger(0, 255)); // Display width out of range for column 'c0' (max = - // 255) + // 255) sb.append(")"); } break; From 33e2865ff25293a20ce1c85b2995fdc21030760c Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Wed, 24 May 2023 02:00:29 +0200 Subject: [PATCH 344/856] Materialize: Fix duplicate table creations (#792) See explanation here: https://github.com/MaterializeInc/materialize/issues/19431#issuecomment-1559695135 Fixes the test issues described in https://github.com/sqlancer/sqlancer/pull/785 --- src/sqlancer/materialize/MaterializeProvider.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java index 3e6e1c59f..f35e476f3 100644 --- a/src/sqlancer/materialize/MaterializeProvider.java +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -216,11 +216,15 @@ public SQLConnection createDatabase(MaterializeGlobalState globalState) throws S globalState.getState().logStatement(String.format("\\c %s;", databaseName)); con = DriverManager.getConnection("jdbc:" + testURL, username, password); - // Serializable transaction isolation is much faster than Strict - // Serializable and should guarantee enough for SQLancer: - // https://materialize.com/docs/overview/isolation-level/ try (Statement s = con.createStatement()) { + // Serializable transaction isolation is much faster than Strict + // Serializable and should guarantee enough for SQLancer: + // https://materialize.com/docs/overview/isolation-level/ s.execute("SET transaction_isolation = 'SERIALIZABLE'"); + // Make sure tables still are visible immediately by not using an + // index for them, see + // https://github.com/MaterializeInc/materialize/issues/19431 + s.execute("SET auto_route_introspection_queries = false"); } return new SQLConnection(con); } From f0c7eedd1532fd9d56252bdd5e76a9ec2e504994 Mon Sep 17 00:00:00 2001 From: ZhengLin Li <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 30 May 2023 09:29:31 +0800 Subject: [PATCH 345/856] Fix yugabyte tablegroups in colocated database (#795) * fix: yugabyte error: cannot use tablegroups in a colocated database * fix: delete useless import * fix: column index starts from 1 * fix: change check colocated logic when creating table * fix: add exception handle * style: check statement will not return an error * fix: remove useless null check * fix: PMD violation --- src/sqlancer/yugabyte/ysql/YSQLSchema.java | 12 ++++++++++++ src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java | 6 ++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/YSQLSchema.java b/src/sqlancer/yugabyte/ysql/YSQLSchema.java index 6aba5ac7d..400a34e21 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLSchema.java +++ b/src/sqlancer/yugabyte/ysql/YSQLSchema.java @@ -160,6 +160,18 @@ protected static List getTableColumns(SQLConnection con, String tabl return columns; } + public boolean getDatabaseIsColocated(SQLConnection con) { + try (Statement s = con.createStatement(); ResultSet rs = s.executeQuery("SELECT yb_is_database_colocated();")) { + rs.next(); + String result = rs.getString(1); + // The query will result in a 'f' for a non-colocated database + return !"f".equals(result); + + } catch (SQLException e) { + throw new AssertionError(e); + } + } + public YSQLTables getRandomTableNonEmptyTables() { return new YSQLTables(Randomly.nonEmptySubset(getDatabaseTables())); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java index 09b559e4c..231bf8008 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLCommon.java @@ -159,8 +159,10 @@ public static void generateWith(StringBuilder sb, YSQLGlobalState globalState, E } } else if (Randomly.getBoolean()) { errors.add("Cannot use TABLEGROUP with TEMP table"); - sb.append(" TABLEGROUP tg").append( - Randomly.getNotCachedInteger(1, (int) YSQLTableGroupGenerator.UNIQUE_TABLEGROUP_COUNTER.get())); + if (!globalState.getSchema().getDatabaseIsColocated(globalState.getConnection())) { + sb.append(" TABLEGROUP tg").append( + Randomly.getNotCachedInteger(1, (int) YSQLTableGroupGenerator.UNIQUE_TABLEGROUP_COUNTER.get())); + } } } From a89d41e5161c272f4ac013adf815f70dc0e94620 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 31 May 2023 11:25:38 +0200 Subject: [PATCH 346/856] ClickHouse: set allow_suspicious_indices (#790) * ClickHouse: set allow_suspicious_indices * Fix table creation * It works locally * another try * add comment --- src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java index 19ce78d27..3df5c5524 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseTableGenerator.java @@ -98,6 +98,8 @@ public void start() { columns.stream().map(c -> c.asColumnReference(null)).collect(Collectors.toList()), 3); sb.append(ClickHouseToStringVisitor.asString(expr)); } + // Suppress index sanity checks https://github.com/sqlancer/sqlancer/issues/788 + sb.append(" SETTINGS allow_suspicious_indices=1"); // TODO: PRIMARY KEY } From 7d894b5afbb3e8e2b3c2494ed3ec472646366375 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 1 Jun 2023 23:24:08 -0400 Subject: [PATCH 347/856] Remove the redundancy in comparing floating-point values (#797) The canonicalization was first introduced as part of https://github.com/sqlancer/sqlancer/pull/497 and then copied to subsequent implementations. --- src/sqlancer/ComparatorHelper.java | 16 ++++++++++++++++ .../test/tlp/DatabendQueryPartitioningBase.java | 10 ---------- .../DatabendQueryPartitioningDistinctTester.java | 2 +- .../DatabendQueryPartitioningGroupByTester.java | 2 +- .../DatabendQueryPartitioningHavingTester.java | 2 +- .../DatabendQueryPartitioningWhereTester.java | 2 +- .../oracle/tlp/DorisQueryPartitioningBase.java | 16 ---------------- .../DorisQueryPartitioningDistinctTester.java | 2 +- .../tlp/DorisQueryPartitioningGroupByTester.java | 2 +- .../tlp/DorisQueryPartitioningHavingTester.java | 2 +- .../tlp/DorisQueryPartitioningWhereTester.java | 2 +- .../duckdb/test/DuckDBQueryPartitioningBase.java | 16 ---------------- .../DuckDBQueryPartitioningDistinctTester.java | 2 +- .../DuckDBQueryPartitioningGroupByTester.java | 2 +- .../DuckDBQueryPartitioningHavingTester.java | 2 +- .../test/DuckDBQueryPartitioningWhereTester.java | 2 +- .../test/QuestDBQueryPartitioningBase.java | 10 ---------- .../QuestDBQueryPartitioningWhereTester.java | 2 +- 18 files changed, 29 insertions(+), 65 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 880d3ae72..c2fc009ef 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -166,4 +166,20 @@ public static List getCombinedResultSetNoDuplicates(String firstQueryStr return secondResultSet; } + public static String canonicalizeResultValue(String value) { + if (value == null) { + return value; + } + + switch (value) { + case "-0.0": + return "0.0"; + case "-0": + return "0"; + default: + } + + return value; + } + } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 28d516f65..0042f83d6 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -43,15 +42,6 @@ public DatabendQueryPartitioningBase(DatabendGlobalState state) { DatabendErrors.addExpressionErrors(errors); } - public static String canonicalizeResultValue(String value) { - // Rule: -0.0 should be canonicalized to 0.0 - if (Objects.equals(value, "-0.0")) { - return "0.0"; - } - - return value; - } - @Override public void check() throws SQLException { s = state.getSchema(); diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java index 5756a00be..430c5cfc6 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java @@ -41,7 +41,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DatabendQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java index cbac6820b..6ec148ae8 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java @@ -42,7 +42,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DatabendQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java index a6a8bf47e..2eda63ab5 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java @@ -54,7 +54,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DatabendQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index cff5ff1dd..a9ac6a6c3 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -40,7 +40,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DatabendQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java index 5db230854..5a40e3134 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java @@ -41,22 +41,6 @@ public DorisQueryPartitioningBase(DorisGlobalState state) { DorisErrors.addInsertErrors(errors); } - public static String canonicalizeResultValue(String value) { - if (value == null) { - return value; - } - - switch (value) { - case "-0.0": - return "0.0"; - case "-0": - return "0"; - default: - } - - return value; - } - @Override public void check() throws SQLException { s = state.getSchema(); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java index 6cc15fdc8..3bce110ff 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java @@ -39,7 +39,7 @@ public void check() throws SQLException { combinedString.add(unionString); List secondResultSet = ComparatorHelper.getResultSetFirstColumnAsString(unionString, errors, state); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DorisQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java index 0761fddb6..00a0cf28a 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java @@ -43,7 +43,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DorisQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java index 7ce10dc09..2151e1da3 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -53,7 +53,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DorisQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java index c45bc36fb..e373e98b9 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -47,7 +47,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DorisQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 837247f62..2acc6a6d6 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -37,22 +37,6 @@ public DuckDBQueryPartitioningBase(DuckDBGlobalState state) { DuckDBErrors.addExpressionErrors(errors); } - public static String canonicalizeResultValue(String value) { - if (value == null) { - return value; - } - - switch (value) { - case "-0.0": - return "0.0"; - case "-0": - return "0"; - default: - } - - return value; - } - @Override public void check() throws SQLException { s = state.getSchema(); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java index d85d2a2d5..70cd731d6 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningDistinctTester.java @@ -38,7 +38,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java index fab480fed..86c26370d 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java @@ -41,7 +41,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, secondQueryString, thirdQueryString, combinedString, true, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index aef003fa5..e8988d867 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -47,7 +47,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java index 412ab6ffd..ca9cde670 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java @@ -39,7 +39,7 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, DuckDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index fa5283507..068abbd38 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -3,7 +3,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -48,15 +47,6 @@ List> generateFetchColumns() { return columns; } - public static String canonicalizeResultValue(String value) { - // Rule: -0.0 should be canonicalized to 0.0 - if (Objects.equals(value, "-0.0")) { - return "0.0"; - } - - return value; - } - @Override protected ExpressionGenerator> getGen() { return gen; diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java index fe5b5890d..ed89e7136 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningWhereTester.java @@ -36,6 +36,6 @@ public void check() throws SQLException { List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, false, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, QuestDBQueryPartitioningBase::canonicalizeResultValue); + state, ComparatorHelper::canonicalizeResultValue); } } From e4184896679662ce4da8e71aa34a4db56cba45fc Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 2 Jun 2023 04:13:11 -0400 Subject: [PATCH 348/856] Fix the package declaration for the Materialize QPG test (#796) --- test/sqlancer/qpg/materialize/TestMaterializeQPG.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQPG.java b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java index 0564d048c..431468cee 100644 --- a/test/sqlancer/qpg/materialize/TestMaterializeQPG.java +++ b/test/sqlancer/qpg/materialize/TestMaterializeQPG.java @@ -1,4 +1,4 @@ -package sqlancer.qpg.cockroachdb; +package sqlancer.qpg.materialize; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; From d013225e5d0d0637022c1d14f2d133f86bbb40a0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 6 Jun 2023 22:19:40 -0400 Subject: [PATCH 349/856] Add a workaround for issue 799 (#800) --- src/sqlancer/Main.java | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 29610f266..dfdc949d6 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -23,9 +23,32 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.JCommander.Builder; +import sqlancer.arangodb.ArangoDBProvider; +import sqlancer.citus.CitusProvider; +import sqlancer.clickhouse.ClickHouseProvider; +import sqlancer.cnosdb.CnosDBProvider; +import sqlancer.cockroachdb.CockroachDBProvider; import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.cosmos.CosmosProvider; +import sqlancer.databend.DatabendProvider; +import sqlancer.doris.DorisProvider; +import sqlancer.duckdb.DuckDBProvider; +import sqlancer.h2.H2Provider; +import sqlancer.hsqldb.HSQLDBProvider; +import sqlancer.mariadb.MariaDBProvider; +import sqlancer.materialize.MaterializeProvider; +import sqlancer.mongodb.MongoDBProvider; +import sqlancer.mysql.MySQLProvider; +import sqlancer.oceanbase.OceanBaseProvider; +import sqlancer.postgres.PostgresProvider; +import sqlancer.questdb.QuestDBProvider; +import sqlancer.sqlite3.SQLite3Provider; +import sqlancer.tidb.TiDBProvider; +import sqlancer.timescaledb.TimescaleDBProvider; +import sqlancer.yugabyte.ycql.YCQLProvider; +import sqlancer.yugabyte.ysql.YSQLProvider; public final class Main { @@ -609,9 +632,41 @@ private boolean run(MainOptions options, ExecutorService execService, for (DatabaseProvider provider : loader) { providers.add(provider); } + checkForIssue799(providers); return providers; } + // see https://github.com/sqlancer/sqlancer/issues/799 + private static void checkForIssue799(List> providers) { + if (providers.isEmpty()) { + System.err.println( + "No DBMS implementations (i.e., instantiations of the DatabaseProvider class) were found. You likely ran into an issue described in https://github.com/sqlancer/sqlancer/issues/799. As a workaround, I now statically load all supported providers as of June 7, 2023."); + providers.add(new ArangoDBProvider()); + providers.add(new CitusProvider()); + providers.add(new ClickHouseProvider()); + providers.add(new CnosDBProvider()); + providers.add(new CockroachDBProvider()); + providers.add(new CosmosProvider()); + providers.add(new DatabendProvider()); + providers.add(new DorisProvider()); + providers.add(new DuckDBProvider()); + providers.add(new H2Provider()); + providers.add(new HSQLDBProvider()); + providers.add(new MariaDBProvider()); + providers.add(new MaterializeProvider()); + providers.add(new MongoDBProvider()); + providers.add(new MySQLProvider()); + providers.add(new OceanBaseProvider()); + providers.add(new PostgresProvider()); + providers.add(new QuestDBProvider()); + providers.add(new SQLite3Provider()); + providers.add(new TiDBProvider()); + providers.add(new TimescaleDBProvider()); + providers.add(new YCQLProvider()); + providers.add(new YSQLProvider()); + } + } + private static synchronized void startProgressMonitor() { if (progressMonitorStarted) { /* From a4e828b72c744d372b653b4154c2a0ed25d5a176 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 13 Jun 2023 11:52:21 +0800 Subject: [PATCH 350/856] fix: rename master to main, add certain constrains (#813) * fix: rename master to main, add certain constrains * feat: add constrains, rename --- .github/workflows/main.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 945d1bc18..3abeeb9e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,10 +3,16 @@ name: ci on: push: branches: - - master + - main + paths: + - 'src/**' + - '.github/workflows/*.yml' pull_request: branches: - - master + - main + paths: + - 'src/**' + - '.github/workflows/*.yml' jobs: misc: From e55fc4448cdbb8bcaf3cfea08d876d0b2b4b319d Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:40:53 +0800 Subject: [PATCH 351/856] refactor: refactor MySQLTableGenerator (#812) --- .../mysql/gen/MySQLTableGenerator.java | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 427f5b7c3..bc0533295 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -18,13 +18,11 @@ import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; public class MySQLTableGenerator { - private final StringBuilder sb = new StringBuilder(); private final boolean allowPrimaryKey; private boolean setPrimaryKey; private final String tableName; private final Randomly r; - private int columnId; private boolean tableHasNullableColumn; private MySQLEngine engine; private int keysSpecified; @@ -65,18 +63,18 @@ private SQLQueryAdapter create() { if (i != 0) { sb.append(", "); } - appendColumn(); + appendColumn(i); } sb.append(")"); sb.append(" "); appendTableOptions(); appendPartitionOptions(); - if ((tableHasNullableColumn || setPrimaryKey) && engine == MySQLEngine.CSV) { + if (engine == MySQLEngine.CSV && (tableHasNullableColumn || setPrimaryKey)) { if (true) { // TODO // results in an error throw new IgnoreMeException(); } - } else if ((tableHasNullableColumn || keysSpecified > 1) && engine == MySQLEngine.ARCHIVE) { + } else if (engine == MySQLEngine.ARCHIVE && (tableHasNullableColumn || keysSpecified > 1)) { errors.add("Too many keys specified; max 1 keys allowed"); errors.add("Table handler doesn't support NULL in given index"); addCommonErrors(errors); @@ -248,27 +246,21 @@ private void appendTableOptions() { } } - private void appendColumn() { + private void appendColumn(int columnId) { String columnName = DBMSCommon.createColumnName(columnId); columns.add(columnName); sb.append(columnName); appendColumnDefinition(); - columnId++; } private enum ColumnOptions { NULL_OR_NOT_NULL, UNIQUE, COMMENT, COLUMN_FORMAT, STORAGE, PRIMARY_KEY } - private void appendColumnDefinition() { - sb.append(" "); - MySQLDataType randomType = MySQLDataType.getRandom(globalState); - boolean isTextType = randomType == MySQLDataType.VARCHAR; - appendTypeString(randomType); - sb.append(" "); + private void appendColumnOption(MySQLDataType type) { + boolean isTextType = type == MySQLDataType.VARCHAR; boolean isNull = false; boolean columnHasPrimaryKey = false; - List columnOptions = Randomly.subset(ColumnOptions.values()); if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { tableHasNullableColumn = true; @@ -324,10 +316,17 @@ private void appendColumnDefinition() { throw new AssertionError(); } } + } + private void appendColumnDefinition() { + sb.append(" "); + MySQLDataType randomType = MySQLDataType.getRandom(globalState); + appendType(randomType); + sb.append(" "); + appendColumnOption(randomType); } - private void appendTypeString(MySQLDataType randomType) { + private void appendType(MySQLDataType randomType) { switch (randomType) { case DECIMAL: sb.append("DECIMAL"); From eb0a65d41a87dc5ad1c7c7a0863d06f8010e3c33 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Fri, 16 Jun 2023 11:56:41 +0800 Subject: [PATCH 352/856] tidb: optimize InsertGenerator (#823) * tidb: optimize InsertGenerator * tidb: add expected errors * tidb: add expected error --- src/sqlancer/tidb/TiDBSchema.java | 1 + .../tidb/gen/TiDBAlterTableGenerator.java | 2 ++ src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 2 ++ .../tidb/gen/TiDBInsertGenerator.java | 22 +++++++++---------- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 1 + 5 files changed, 17 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 609d1dbe7..265cacbda 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -239,6 +239,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { break; case "double": case "double(8,6)": // workaround to address https://github.com/sqlancer/sqlancer/issues/669 + case "double(23,16)": size = 8; primitiveType = TiDBDataType.FLOATING; break; diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index b6f65a780..6a95fb9a9 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -30,6 +30,8 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("supported"); errors.add("SQL syntax"); errors.add("can't drop"); + errors.add("A PRIMARY must include all columns in the table's partitioning function"); + errors.add("key was too long"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index cb870a2df..1be2753b6 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -61,6 +61,8 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx } errors.add("Cannot decode index value, because"); // invalid value for generated column errors.add("index already exist"); + errors.add("Data truncation"); + errors.add("key was too long"); return new SQLQueryAdapter(sb.toString(), errors, true); } diff --git a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java index a78d6c2fb..363fa438b 100644 --- a/src/sqlancer/tidb/gen/TiDBInsertGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBInsertGenerator.java @@ -26,11 +26,15 @@ public TiDBInsertGenerator(TiDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - return new TiDBInsertGenerator(globalState).get(); + TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + return new TiDBInsertGenerator(globalState).get(table); } - private SQLQueryAdapter get() { - TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState, TiDBTable table) { + return new TiDBInsertGenerator(globalState).get(table); + } + + private SQLQueryAdapter get(TiDBTable table) { gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); StringBuilder sb = new StringBuilder(); boolean isInsert = Randomly.getBoolean(); @@ -75,18 +79,14 @@ private void insertColumns(StringBuilder sb, List columns) { sb.append(", "); } sb.append("("); - for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { - if (nrColumn != 0) { + int i = 0; + for (TiDBColumn c : columns) { + if (i++ != 0) { sb.append(", "); } - insertValue(sb); + sb.append(TiDBVisitor.asString(gen.generateConstant(c.getType().getPrimitiveDataType()))); } sb.append(")"); } } - - private void insertValue(StringBuilder sb) { - sb.append(gen.generateConstant()); // TODO: try to insert valid data - } - } diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index 4349b69ca..af9ad4457 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -33,6 +33,7 @@ public static SQLQueryAdapter createRandomTableStatement(TiDBGlobalState globalS public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { errors.add("Information schema is changed during the execution of the statement"); + errors.add("A CLUSTERED INDEX must include all columns in the table's partitioning function"); String tableName = globalState.getSchema().getFreeTableName(); int nrColumns = Randomly.smallNumber() + 1; allowPrimaryKey = Randomly.getBoolean(); From 0f9eef728108088615318aa33a821be22ea1177a Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Fri, 16 Jun 2023 16:30:53 +0800 Subject: [PATCH 353/856] feat: Add virtual db for statement reduction testing (#815) * feat: Add virtual db for statement reduction testing + Implement a prototype of `VirtualDB` so that statement reduction can be tested. + Write some test cases for statement reduction. + Simplify the code of StatementReducer + Add parameter for timeout and max steps of reducer * Correct code of VirtualDB. * Add an extra empty line for StatementReducer.java * Add GitHub Action for reducer tests * format * format 2 * Fix name issues * Do not run reducer test in "general" --------- Co-authored-by: Yichen Yan Co-authored-by: Yichen Yan --- .github/workflows/main.yml | 20 ++- src/sqlancer/MainOptions.java | 15 +++ src/sqlancer/StatementReducer.java | 69 ++++++++--- test/sqlancer/reducer/TestEnvironment.java | 114 ++++++++++++++++++ .../reducer/TestStatementReducer.java | 75 ++++++++++++ .../VirtualDB/VirtualDBConnection.java | 18 +++ .../reducer/VirtualDB/VirtualDBErrors.java | 12 ++ .../VirtualDB/VirtualDBGlobalState.java | 56 +++++++++ .../reducer/VirtualDB/VirtualDBOptions.java | 29 +++++ .../reducer/VirtualDB/VirtualDBProvider.java | 54 +++++++++ .../reducer/VirtualDB/VirtualDBQuery.java | 30 +++++ .../reducer/VirtualDB/VirtualDBSchema.java | 46 +++++++ 12 files changed, 522 insertions(+), 16 deletions(-) create mode 100644 test/sqlancer/reducer/TestEnvironment.java create mode 100644 test/sqlancer/reducer/TestStatementReducer.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java create mode 100644 test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3abeeb9e2..1bafaf3d1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -30,7 +30,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test + run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**,!sqlancer.reducer.**' test - name: Set up Python uses: actions/setup-python@v4 with: @@ -533,3 +533,21 @@ jobs: run: mvn -B package -DskipTests=true - name: Shortly run DuckDB run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb + + reducer: + name: Reducer Tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v1.4.4 + with: + java-version: 11 + - name: Build + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + mvn -Dtest=TestStatementReducer test diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 5283ea2a6..c653a358f 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -10,6 +10,7 @@ @Parameters(separators = "=", commandDescription = "Options applicable to all DBMS") public class MainOptions { public static final int NO_SET_PORT = -1; + public static final int NO_REDUCE_LIMIT = -1; public static final MainOptions DEFAULT_OPTIONS = new MainOptions(); @Parameter(names = { "--help", "-h" }, description = "Lists all supported options and commands", help = true) @@ -125,6 +126,12 @@ public class MainOptions { @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") private boolean useReducer = false; // NOPMD + @Parameter(names = "--statement-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the statement reducer will do") + private long maxStatementReduceSteps = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--statement-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the statement reducer will do") + private long maxStatementReduceTime = NO_REDUCE_LIMIT; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -285,4 +292,12 @@ public boolean performConnectionTest() { public boolean useReducer() { return useReducer; } + + public long getMaxStatementReduceSteps() { + return maxStatementReduceSteps; + } + + public long getMaxStatementReduceTime() { + return maxStatementReduceTime; + } } diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java index 14db4ba5b..3d3d36711 100644 --- a/src/sqlancer/StatementReducer.java +++ b/src/sqlancer/StatementReducer.java @@ -1,9 +1,9 @@ package sqlancer; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.function.BiFunction; -import java.util.stream.Collectors; import sqlancer.common.query.Query; @@ -11,49 +11,85 @@ public class StatementReducer, O extends DBMSSpec implements Reducer { private final DatabaseProvider provider; private boolean observedChange; + private int partitionNum; public StatementReducer(DatabaseProvider provider) { this.provider = provider; } + private boolean hasNotReachedLimit(long curr, long limit) { + if (limit == MainOptions.NO_REDUCE_LIMIT) { + return true; + } + return curr < limit; + } + @SuppressWarnings("unchecked") @Override public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + long maxReduceTime = state.getOptions().getMaxStatementReduceTime(); + long maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); + List> knownToReproduceBugStatements = new ArrayList<>(); for (Query stat : state.getState().getStatements()) { knownToReproduceBugStatements.add((Query) stat); } + System.out.println("Starting query:"); printQueries(knownToReproduceBugStatements); System.out.println(); - do { + if (knownToReproduceBugStatements.size() <= 1) { + return; + } + + Instant timeOfReductionBegins = Instant.now(); + long currentReduceSteps = 0; + long currentReduceTime = 0; + partitionNum = 2; + + while (knownToReproduceBugStatements.size() >= 2 && hasNotReachedLimit(currentReduceSteps, maxReduceSteps) + && hasNotReachedLimit(currentReduceTime, maxReduceTime)) { observedChange = false; + knownToReproduceBugStatements = tryReduction(state, reproducer, newGlobalState, - knownToReproduceBugStatements, (candidateStatements, i) -> { - candidateStatements.remove((int) i); - return true; - }); - } while (observedChange); + knownToReproduceBugStatements); + if (!observedChange) { + if (partitionNum == knownToReproduceBugStatements.size()) { + break; + } + // increase the search granularity + partitionNum = Math.min(partitionNum * 2, knownToReproduceBugStatements.size()); + + currentReduceSteps++; + Instant currentInstant = Instant.now(); + currentReduceTime = Duration.between(currentInstant, timeOfReductionBegins).getSeconds(); + } + + } System.out.println("Reduced query:"); printQueries(knownToReproduceBugStatements); + newGlobalState.getState().setStatements(new ArrayList<>(knownToReproduceBugStatements)); } private List> tryReduction(G state, // NOPMD - Reproducer reproducer, G newGlobalState, List> knownToReproduceBugStatements, - BiFunction>, Integer, Boolean> reductionOperation) throws Exception { + Reproducer reproducer, G newGlobalState, List> knownToReproduceBugStatements) throws Exception { List> statements = knownToReproduceBugStatements; - for (int i = 0; i < statements.size(); i++) { + + int start = 0; + int subLength = statements.size() / partitionNum; + while (start < statements.size()) { + // newStatements = candidate[:start] + candidate[start+subLength:] + // in other word, remove [start, start+subLength) from candidates try (C con2 = provider.createDatabase(newGlobalState)) { newGlobalState.setConnection(con2); List> candidateStatements = new ArrayList<>(statements); - if (!reductionOperation.apply(candidateStatements, i)) { - continue; - } - newGlobalState.getState().setStatements(candidateStatements.stream().collect(Collectors.toList())); + candidateStatements.subList(start, start + subLength).clear(); + newGlobalState.getState().setStatements(new ArrayList<>(candidateStatements)); + for (Query s : candidateStatements) { try { s.execute(newGlobalState); @@ -65,6 +101,8 @@ private List> tryReduction(G state, // NOPMD if (reproducer.bugStillTriggers(newGlobalState)) { observedChange = true; statements = candidateStatements; + partitionNum = Math.max(partitionNum - 1, 2); + break; // reproducer.outputHook((SQLite3GlobalState) newGlobalState); // state.getLogger().logReduced(newGlobalState.getState()); } @@ -72,6 +110,7 @@ private List> tryReduction(G state, // NOPMD } } + start = start + subLength; } return statements; } diff --git a/test/sqlancer/reducer/TestEnvironment.java b/test/sqlancer/reducer/TestEnvironment.java new file mode 100644 index 000000000..3329597aa --- /dev/null +++ b/test/sqlancer/reducer/TestEnvironment.java @@ -0,0 +1,114 @@ +package sqlancer.reducer; + +import sqlancer.*; +import sqlancer.common.query.Query; +import sqlancer.reducer.VirtualDB.VirtualDBGlobalState; +import sqlancer.reducer.VirtualDB.VirtualDBProvider; +import sqlancer.reducer.VirtualDB.VirtualDBQuery; + +import java.util.ArrayList; +import java.util.List; +import java.util.ServiceLoader; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * TODO: Make Connection a generic type OR Fake a conn QUERY AND CONNECTION BOTH ARE FAKE. FAKE QUERY sub class + */ +public class TestEnvironment { + private final String databaseName = "virtual_db"; + private final MainOptions options = new MainOptions(); + private VirtualDBProvider provider = null; + private VirtualDBGlobalState state, newGlobalState; + + public TestEnvironment() throws Exception { + setUpTestingEnvironment(); + } + + /** + * @param queries: + * List of Query + * + * @return String of queries that appended together with '\n' separated (no '\n' at the last line) + */ + public static String getQueriesString(List> queries) { + return queries.stream().map(Query::getQueryString).collect(Collectors.joining("\n")); + } + + private VirtualDBGlobalState createGlobalState() { + try { + return provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + } catch (Exception e) { + throw new AssertionError(e); + } + } + + @SuppressWarnings("rawtypes") + private void initVirtualDBProvider() { + try { + ServiceLoader loader = ServiceLoader.load(DatabaseProvider.class); + for (DatabaseProvider provider : loader) { + if (provider.getDBMSName().equals(databaseName)) { + this.provider = (VirtualDBProvider) provider; + break; + } + } + if (provider == null) { + throw new AssertionError("testing provider not registered"); + } + } catch (Exception e) { + throw new AssertionError(e); + } + + } + + private void setUpTestingEnvironment() throws Exception { + initVirtualDBProvider(); + state = createGlobalState(); + StateToReproduce stateToReproduce = provider.getStateToReproduce(databaseName); + + state.setState(stateToReproduce); + state.setDatabaseName(databaseName); + state.setMainOptions(options); + + // Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + // state.setStateLogger(logger); + + try (SQLConnection con = provider.createDatabase(state)) { + state.setConnection(con); + newGlobalState = createGlobalState(); + // Main.StateLogger newLogger = new Main.StateLogger(databaseName, provider, options); + // newGlobalState.setStateLogger(newLogger); + newGlobalState.setState(stateToReproduce); + newGlobalState.setDatabaseName(databaseName); + newGlobalState.setMainOptions(options); + } + } + + public void setInitialStatementsFromStrings(List statements) { + List> queries = new ArrayList<>(); + for (String s : statements) { + queries.add(new VirtualDBQuery(s)); + } + state.getState().setStatements(queries); + } + + public void setBugInducingCondition(Function>, Boolean> bugInducingCondition) { + state.setBugInducingCondition(bugInducingCondition); + newGlobalState.setBugInducingCondition(bugInducingCondition); + } + + public void runReduce() throws Exception { + Reducer reducer = new StatementReducer<>(provider); + Reproducer reproducer = provider.generateAndTestDatabase(newGlobalState); + reducer.reduce(state, reproducer, newGlobalState); + } + + public List> getReducedStatements() { + return newGlobalState.getState().getStatements(); + } + + public List> getInitialStatements() { + return state.getState().getStatements(); + } +} diff --git a/test/sqlancer/reducer/TestStatementReducer.java b/test/sqlancer/reducer/TestStatementReducer.java new file mode 100644 index 000000000..83c7de94e --- /dev/null +++ b/test/sqlancer/reducer/TestStatementReducer.java @@ -0,0 +1,75 @@ +package sqlancer.reducer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.query.Query; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class TestStatementReducer { + + @Test + void testSimple() throws Exception { + TestEnvironment env = new TestEnvironment(); + + String[] queriesStr = { "CREATE TABLE FAKE_TABLE;", "SELECT * FROM FAKE_TABLE;", "EXIT", }; + env.setInitialStatementsFromStrings(List.of(queriesStr)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("SELECT"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals(1, reducedResult.size()); + assertEquals("SELECT * FROM FAKE_TABLE;", reducedResult.get(0).toString()); + + } + + @Test + void testDeltaDebugging() throws Exception { + TestEnvironment env = new TestEnvironment(); + List fakeStatements = new ArrayList<>(); + for (int i = 0; i < 10000; i++) { + String statement = "Statement_" + i + ";"; + fakeStatements.add(statement); + } + + env.setInitialStatementsFromStrings(fakeStatements); + env.setBugInducingCondition(statements -> { + String queries = TestEnvironment.getQueriesString(statements); + return queries.contains("Statement_29;"); + }); + + env.runReduce(); + List> reducedQueries = env.getReducedStatements(); + String queriesString = TestEnvironment.getQueriesString(reducedQueries); + assertEquals(queriesString, "Statement_29;"); + } + + @Test + void testDeltaDebuggingWithStatementsCombination() throws Exception { + TestEnvironment env = new TestEnvironment(); + List fakeStatements = new ArrayList<>(); + + String pattern = "(.*\\n)*(Statement_2;)\\n(.*\\n)*(Statement_318);\\n(.*\\n)*(Statement_990;)(.*\\n)*.*"; + for (int i = 0; i < 1000; i++) { + String statement = "Statement_" + i + ";"; + fakeStatements.add(statement); + } + + env.setInitialStatementsFromStrings(fakeStatements); + env.setBugInducingCondition(queryList -> { + String queries = TestEnvironment.getQueriesString(queryList); + return Pattern.matches(pattern, queries); + }); + + env.runReduce(); + List> reducedQueries = env.getReducedStatements(); + String queriesString = TestEnvironment.getQueriesString(reducedQueries); + assertEquals(queriesString, "Statement_2;\nStatement_318;\nStatement_990;"); + } + +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java b/test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java new file mode 100644 index 000000000..6a562a90d --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBConnection.java @@ -0,0 +1,18 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.SQLConnection; + +import java.sql.Connection; +import java.sql.SQLException; + +public class VirtualDBConnection extends SQLConnection { + + public VirtualDBConnection(Connection connection) { + super(connection); + } + + @Override + public void close() throws SQLException { + + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java b/test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java new file mode 100644 index 000000000..64c783c11 --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBErrors.java @@ -0,0 +1,12 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.common.query.ExpectedErrors; + +public final class VirtualDBErrors { + public VirtualDBErrors() { + } + + public static void addErrors(ExpectedErrors errors) { + errors.add("Default error"); + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java b/test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java new file mode 100644 index 000000000..a0548495c --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBGlobalState.java @@ -0,0 +1,56 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.Query; + +import java.util.List; +import java.util.function.Function; + +@SuppressWarnings("all") +public class VirtualDBGlobalState extends SQLGlobalState { + + private SQLConnection virtualConn = new SQLConnection(null); + private StringBuilder queriesStringBuilder = new StringBuilder(); + private Function>, Boolean> bugInducingCondition = null; + + public Function>, Boolean> getBugInducingCondition() { + return bugInducingCondition; + } + + public void setBugInducingCondition(Function>, Boolean> condition) { + bugInducingCondition = (condition); + } + + @Override + protected VirtualDBSchema readSchema() throws Exception { + return null; + } + + @Override + public SQLConnection getConnection() { + // It's a fake engine, so the connection would not be available :) + return virtualConn; + } + + @Override + public void setConnection(SQLConnection con) { + // A fake connection could also not be closed. + // So nothing would be done here. + // And reset the query String (Seems needless) + // queriesStringBuilder = new StringBuilder(); + } + + // public String getCurrentQueriesString() { + // return queriesStringBuilder.toString(); + // } + + @Override + public boolean executeStatement(Query q, String... fills) throws Exception { + if (queriesStringBuilder.length() != 0) { + queriesStringBuilder.append("\n"); + } + queriesStringBuilder.append(q.getQueryString()); + return true; + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java b/test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java new file mode 100644 index 000000000..72cbe918c --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBOptions.java @@ -0,0 +1,29 @@ +package sqlancer.reducer.VirtualDB; + +import com.beust.jcommander.Parameters; +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.reducer.VirtualDB.VirtualDBOptions.VirtualDBFactory; + +import java.util.ArrayList; +import java.util.List; + +@Parameters(separators = "=", commandDescription = "VirtualDB (default port: " + "-1" + ", default host: " + "127.0.0.1" + + ")") +public class VirtualDBOptions implements DBMSSpecificOptions { + + List factories = new ArrayList<>(); + + @Override + public List getTestOracleFactory() { + return factories; + } + + public static class VirtualDBFactory implements OracleFactory { + @Override + public TestOracle create(VirtualDBGlobalState globalState) throws Exception { + return null; + } + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java b/test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java new file mode 100644 index 000000000..697e5c60e --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBProvider.java @@ -0,0 +1,54 @@ +package sqlancer.reducer.VirtualDB; + +import com.google.auto.service.AutoService; +import sqlancer.DatabaseProvider; +import sqlancer.Reproducer; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; + +@AutoService(DatabaseProvider.class) +public class VirtualDBProvider extends SQLProviderAdapter { + + private Reproducer reproducerForTesting; + + public VirtualDBProvider() { + super(VirtualDBGlobalState.class, VirtualDBOptions.class); + } + + @Override + public SQLConnection createDatabase(VirtualDBGlobalState globalState) throws Exception { + return new VirtualDBConnection(null); + } + + @Override + public String getDBMSName() { + return "virtual_db"; + } + + @Override + public void generateDatabase(VirtualDBGlobalState globalState) throws Exception { + + } + + @Override + public Reproducer generateAndTestDatabase(VirtualDBGlobalState globalState) throws Exception { + return state -> { + if (globalState.getBugInducingCondition() == null) + return false; + return globalState.getBugInducingCondition().apply(globalState.getState().getStatements()); + }; + } + + @Override + public Class getGlobalStateClass() { + return super.getGlobalStateClass(); + } + + public Reproducer getReproducerForTesting() { + return reproducerForTesting; + } + + public void setReproducerForTesting(Reproducer reproducer) { + this.reproducerForTesting = reproducer; + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java new file mode 100644 index 000000000..37389109f --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java @@ -0,0 +1,30 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.GlobalState; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; + +import java.sql.SQLException; + +public class VirtualDBQuery extends SQLQueryAdapter { + + public VirtualDBQuery(String query) { + // Since the base class must check the format + // We judge if the statement could affect schema. A bit hacky tho. + super(query, (query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN"))); + } + + public VirtualDBQuery(String query, boolean couldAffectSchema) { + super(query, couldAffectSchema); + } + + @Override + public > boolean execute(G globalState, String... fills) + throws SQLException { + try { + return globalState.executeStatement(this, fills); + } catch (Exception e) { + throw new SQLException(e); + } + } +} diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java b/test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java new file mode 100644 index 000000000..9ea5b422e --- /dev/null +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBSchema.java @@ -0,0 +1,46 @@ +package sqlancer.reducer.VirtualDB; + +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.reducer.VirtualDB.VirtualDBSchema.VirtualDBTable; + +import java.util.List; + +public class VirtualDBSchema extends AbstractSchema { + + public VirtualDBSchema(List databaseTables) { + super(databaseTables); + } + + public static class VirtualDBTable extends AbstractTable { + protected VirtualDBTable(String name, List columns, List indexes, + boolean isView) { + super(name, columns, indexes, isView); + } + + @Override + public long getNrRows(VirtualDBGlobalState globalState) { + return 0; + } + } + + public static class VirtualDBIndex extends TableIndex { + + protected VirtualDBIndex(String indexName) { + super(indexName); + } + } + + public static class VirtualDBDataType { + + } + + public static class VirtualDBColumn extends AbstractTableColumn { + + public VirtualDBColumn(String name, VirtualDBTable table, VirtualDBDataType type) { + super(name, table, type); + } + } +} From 10d603f074293ee203d04461cebd848e3ae3a43a Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Jun 2023 11:39:29 +0800 Subject: [PATCH 354/856] tidb: support cross join and on condition (#827) * tidb: optimize InsertGenerator * tidb: add expected errors * tidb: add expected error * tidb: cross join and on condition * tidb: bypass bug 44747 --- src/sqlancer/tidb/TiDBBugs.java | 3 +++ src/sqlancer/tidb/TiDBErrors.java | 3 +++ src/sqlancer/tidb/ast/TiDBJoin.java | 21 ++++++++++++++++--- .../tidb/visitor/TiDBToStringVisitor.java | 14 ++++++------- 4 files changed, 30 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index d56629e44..4aef122e3 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -14,6 +14,9 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/38295 public static boolean bug38295 = true; + // https://github.com/pingcap/tidb/issues/44747 + public static boolean bug44747 = true; + private TiDBBugs() { } diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index c2db11757..2068f9f1e 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -52,6 +52,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (TiDBBugs.bug38295) { errors.add("assertion failed"); } + if (TiDBBugs.bug44747) { + errors.add("index out of range"); + } } public static void addExpressionHavingErrors(ExpectedErrors errors) { diff --git a/src/sqlancer/tidb/ast/TiDBJoin.java b/src/sqlancer/tidb/ast/TiDBJoin.java index e22b34b6a..dc78b0020 100644 --- a/src/sqlancer/tidb/ast/TiDBJoin.java +++ b/src/sqlancer/tidb/ast/TiDBJoin.java @@ -12,12 +12,12 @@ public class TiDBJoin implements TiDBExpression { private final TiDBExpression leftTable; private final TiDBExpression rightTable; - private final JoinType joinType; - private final TiDBExpression onCondition; + private JoinType joinType; + private TiDBExpression onCondition; private NaturalJoinType outerType; public enum JoinType { - INNER, NATURAL, STRAIGHT, LEFT, RIGHT; + NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; public static JoinType getRandom() { return Randomly.fromOptions(values()); @@ -52,10 +52,18 @@ public JoinType getJoinType() { return joinType; } + public void setJoinType(JoinType joinType) { + this.joinType = joinType; + } + public TiDBExpression getOnCondition() { return onCondition; } + public static TiDBJoin createCrossJoin(TiDBExpression left, TiDBExpression right, TiDBExpression onClause) { + return new TiDBJoin(left, right, JoinType.CROSS, onClause); + } + public static TiDBJoin createNaturalJoin(TiDBExpression left, TiDBExpression right, NaturalJoinType type) { TiDBJoin tiDBJoin = new TiDBJoin(left, right, JoinType.NATURAL, null); tiDBJoin.setNaturalJoinType(type); @@ -110,6 +118,9 @@ public static List getJoins(List tableList, TiDB case RIGHT: joinExpressions.add(TiDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); break; + case CROSS: + joinExpressions.add(TiDBJoin.createCrossJoin(leftTable, rightTable, null)); + break; default: throw new AssertionError(); } @@ -117,4 +128,8 @@ public static List getJoins(List tableList, TiDB return joinExpressions; } + public void setOnCondition(TiDBExpression generateExpression) { + this.onCondition = generateExpression; + } + } diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index 8bd47ed70..e9c199407 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -95,11 +95,7 @@ public void visit(TiDBJoin join) { sb.append(" "); switch (join.getJoinType()) { case INNER: - if (Randomly.getBoolean()) { - sb.append("INNER "); - } else { - sb.append("CROSS "); - } + sb.append("INNER "); sb.append("JOIN "); break; case LEFT: @@ -135,13 +131,15 @@ public void visit(TiDBJoin join) { } sb.append("JOIN "); break; + case CROSS: + sb.append("CROSS JOIN "); + break; default: throw new AssertionError(); } visit(join.getRightTable()); - sb.append(" "); - if (join.getJoinType() != JoinType.NATURAL) { - sb.append("ON "); + if (join.getOnCondition() != null && join.getJoinType() != JoinType.NATURAL) { + sb.append(" ON "); visit(join.getOnCondition()); } } From 34355e6e32a24613a23459dc76dc5daf8b35beeb Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 17 Jun 2023 22:57:55 +0800 Subject: [PATCH 355/856] mysql: bypass 111471 (#828) * mysql: bypass 111471 * formatting --- src/sqlancer/mysql/MySQLBugs.java | 3 +++ src/sqlancer/mysql/MySQLErrors.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 9cd87614d..ea44c83d4 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -19,6 +19,9 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=99135 public static boolean bug99135 = true; + // https://bugs.mysql.com/bug.php?id=111471 + public static boolean bug111471 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 39aa33e1e..540a5c5db 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -10,6 +10,10 @@ private MySQLErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) errors.add("is not valid for CHARACTER SET"); + + if (MySQLBugs.bug111471) { + errors.add("Memory capacity exceeded"); + } } } From 0de1c3757f96d9b5f9c5ecc2f1026bb5c8998cf3 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 18 Jun 2023 15:28:16 +0800 Subject: [PATCH 356/856] tidb: enable non-empty tables (#829) * tidb: enable non-empty tables * typo --- src/sqlancer/tidb/TiDBProvider.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 91223a246..cf5421e7c 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -5,6 +5,8 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -21,6 +23,7 @@ import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.gen.TiDBAlterTableGenerator; import sqlancer.tidb.gen.TiDBAnalyzeTableGenerator; import sqlancer.tidb.gen.TiDBDeleteGenerator; @@ -84,6 +87,7 @@ private static int mapActions(TiDBGlobalState globalState, Action a) { case CREATE_INDEX: return r.getInteger(0, 2); case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); case TRUNCATE: case DELETE: case ADMIN_CHECKSUM_TABLE: @@ -213,4 +217,15 @@ protected void executeMutator(int index, TiDBGlobalState globalState) throws Exc globalState.executeStatement(queryMutateTable); } + @Override + public boolean addRowsToAllTables(TiDBGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (TiDBTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = TiDBInsertGenerator.getQuery(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } From 6ebbaae57b56f492ee728adb711a218631939aed Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Wed, 21 Jun 2023 19:51:45 +0800 Subject: [PATCH 357/856] tidb: cert init (#831) * tidb: utilities for CERT * tidb: CERT init * tidb: formatting * tidb: disable natural join for CERT * tidb: enum in CERT * rename editDistance * tidb: check oracle num * wrap editDistance --- src/sqlancer/common/DBMSCommon.java | 27 ++ src/sqlancer/common/ast/SelectBase.java | 8 + src/sqlancer/tidb/TiDBOptions.java | 12 + src/sqlancer/tidb/TiDBProvider.java | 18 ++ src/sqlancer/tidb/ast/TiDBJoin.java | 33 +++ src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 293 +++++++++++++++++++ 6 files changed, 391 insertions(+) create mode 100644 src/sqlancer/tidb/oracle/TiDBCERTOracle.java diff --git a/src/sqlancer/common/DBMSCommon.java b/src/sqlancer/common/DBMSCommon.java index f302d48ad..4478be2b3 100644 --- a/src/sqlancer/common/DBMSCommon.java +++ b/src/sqlancer/common/DBMSCommon.java @@ -1,5 +1,6 @@ package sqlancer.common; +import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -39,4 +40,30 @@ public static int getMaxIndexInDoubleArray(double... doubleArray) { } return maxIndex; } + + public static boolean areQueryPlanSequencesSimilar(List list1, List list2) { + return editDistance(list1, list2) <= 1; + } + + public static int editDistance(List list1, List list2) { + int[][] dp = new int[list1.size() + 1][list2.size() + 1]; + for (int i = 0; i <= list1.size(); i++) { + for (int j = 0; j <= list2.size(); j++) { + if (i == 0) { + dp[i][j] = j; + } else if (j == 0) { + dp[i][j] = i; + } else { + dp[i][j] = Math.min(dp[i - 1][j - 1] + costOfSubstitution(list1.get(i - 1), list2.get(j - 1)), + Math.min(dp[i - 1][j] + 1, dp[i][j - 1] + 1)); + } + } + } + return dp[list1.size()][list2.size()]; + } + + private static int costOfSubstitution(String string, String string2) { + return string.equals(string2) ? 0 : 1; + } + } diff --git a/src/sqlancer/common/ast/SelectBase.java b/src/sqlancer/common/ast/SelectBase.java index 341b00209..9e02a7f61 100644 --- a/src/sqlancer/common/ast/SelectBase.java +++ b/src/sqlancer/common/ast/SelectBase.java @@ -50,6 +50,10 @@ public void setGroupByExpressions(List groupByExpressions) { this.groupByExpressions = groupByExpressions; } + public void clearGroupByExpressions() { + this.groupByExpressions = Collections.emptyList(); + } + public List getGroupByExpressions() { assert groupByExpressions != null; return groupByExpressions; @@ -83,6 +87,10 @@ public T getHavingClause() { return havingClause; } + public void clearHavingClause() { + this.havingClause = null; + } + public void setLimitClause(T limitClause) { this.limitClause = limitClause; } diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 6fbab6ba8..7ff20027d 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -14,6 +14,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.oracle.TiDBCERTOracle; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @@ -56,6 +57,17 @@ public TestOracle create(TiDBGlobalState globalState) throws SQ oracles.add(new TiDBTLPHavingOracle(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + CERT { + @Override + public TestOracle create(TiDBGlobalState globalState) throws SQLException { + return new TiDBCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } }; } diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index cf5421e7c..3701a4006 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -19,6 +19,7 @@ import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; @@ -35,6 +36,7 @@ import sqlancer.tidb.gen.TiDBTableGenerator; import sqlancer.tidb.gen.TiDBUpdateGenerator; import sqlancer.tidb.gen.TiDBViewGenerator; +import sqlancer.tidb.oracle.TiDBCERTOracle; @AutoService(DatabaseProvider.class) public class TiDBProvider extends SQLProviderAdapter { @@ -136,6 +138,22 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { throw new AssertionError(e); } } + + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 && globalState + .getDbmsSpecificOptions().getTestOracleFactory().get(0).create(globalState) instanceof TiDBCERTOracle) { + // Disable strict Group By constraints for ROW oracle + globalState.executeStatement(new SQLQueryAdapter( + "SET @@sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';")); + + // Enfore statistic collected for all tables + ExpectedErrors errors = new ExpectedErrors(); + TiDBErrors.addExpressionErrors(errors); + for (TiDBTable table : globalState.getSchema().getDatabaseTables()) { + if (!table.isView()) { + globalState.executeStatement(new SQLQueryAdapter("ANALYZE TABLE " + table.getName() + ";", errors)); + } + } + } } @Override diff --git a/src/sqlancer/tidb/ast/TiDBJoin.java b/src/sqlancer/tidb/ast/TiDBJoin.java index dc78b0020..e48c7a6f7 100644 --- a/src/sqlancer/tidb/ast/TiDBJoin.java +++ b/src/sqlancer/tidb/ast/TiDBJoin.java @@ -128,6 +128,39 @@ public static List getJoins(List tableList, TiDB return joinExpressions; } + public static List getJoinsWithoutNature(List tableList, + TiDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBoolean()) { + TiDBTableReference leftTable = (TiDBTableReference) tableList.remove(0); + TiDBTableReference rightTable = (TiDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + TiDBExpressionGenerator joinGen = new TiDBExpressionGenerator(globalState).setColumns(columns); + switch (TiDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(TiDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case STRAIGHT: + joinExpressions.add(TiDBJoin.createStraightJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case LEFT: + joinExpressions.add(TiDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions.add(TiDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + case CROSS: + joinExpressions.add(TiDBJoin.createCrossJoin(leftTable, rightTable, null)); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + public void setOnCondition(TiDBExpression generateExpression) { this.onCondition = generateExpression; } diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java new file mode 100644 index 000000000..d23789b26 --- /dev/null +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -0,0 +1,293 @@ +package sqlancer.tidb.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.tidb.TiDBErrors; +import sqlancer.tidb.TiDBExpressionGenerator; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBDataType; +import sqlancer.tidb.TiDBSchema.TiDBTables; +import sqlancer.tidb.ast.TiDBBinaryLogicalOperation; +import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; +import sqlancer.tidb.ast.TiDBColumnReference; +import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBJoin.JoinType; +import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableReference; +import sqlancer.tidb.visitor.TiDBVisitor; + +public class TiDBCERTOracle implements TestOracle { + private final TiDBGlobalState state; + private TiDBExpressionGenerator gen; + private final ExpectedErrors errors = new ExpectedErrors(); + private TiDBSelect select; + private List queryPlan1Sequences; + private List queryPlan2Sequences; + + public enum Mutator { + JOIN, WHERE, GROUPBY, HAVING, AND, OR, LIMIT; + + public static Mutator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public TiDBCERTOracle(TiDBGlobalState globalState) { + state = globalState; + TiDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + queryPlan1Sequences = new ArrayList<>(); + queryPlan2Sequences = new ArrayList<>(); + + // Randomly generate a query + TiDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new TiDBExpressionGenerator(state).setColumns(tables.getColumns()); + select = new TiDBSelect(); + + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new TiDBColumnReference(c)) + .collect(Collectors.toList())); + select.setFetchColumns(fetchColumns); + + List tableList = tables.getTables().stream().map(t -> new TiDBTableReference(t)) + .collect(Collectors.toList()); + List joins = TiDBJoin.getJoinsWithoutNature(tableList, state); + select.setJoinList(joins); + select.setFromList(tableList); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(select.getFetchColumns()); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBoolean()) { + select.setLimitClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(gen.generateExpression()); + } + + // Get the result of the first query + String queryString1 = TiDBVisitor.asString(select); + double rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + // Mutate the query + boolean increase = false; + + Mutator mutation = Mutator.getRandom(); + switch (mutation) { + case JOIN: + increase = mutateJoin(); + break; + case WHERE: + increase = mutateWhere(); + break; + case GROUPBY: + increase = mutateGroupBy(); + break; + case HAVING: + increase = mutateHaving(); + break; + case AND: + increase = mutateAnd(); + break; + case OR: + increase = mutateOr(); + break; + case LIMIT: + increase = mutateLimit(); + break; + default: + throw new AssertionError(); + } + + // Get the result of the second query + String queryString2 = TiDBVisitor.asString(select); + double rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + // Check structural equivalence + if (!DBMSCommon.areQueryPlanSequencesSimilar(queryPlan1Sequences, queryPlan2Sequences)) { + return; + } + + /* + * https://github.com/pingcap/tidb/issues/38474 A minor issue in TiDB that some operations would round the + * result while others would not. A false alarm happens when both queries have the same number of estimated rows + * but the restrictued one rounds up to a bigger number. To avoid this issue until it is fixed, we make sure the + * gap between two estimated rows is at least 1. + */ + // Check the results + if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { + throw new AssertionError("Mutator: " + mutation + ", Inconsistent result for query: EXPLAIN " + queryString1 + + "; --" + rowCount1 + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + } + } + + private boolean mutateJoin() { + if (select.getJoinList().isEmpty()) { + return false; + } + TiDBJoin join = (TiDBJoin) Randomly.fromList(select.getJoinList()); + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((TiDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((TiDBTableReference) join.getRightTable()).getTable().getColumns()); + TiDBExpressionGenerator joinGen2 = new TiDBExpressionGenerator(state).setColumns(columns); + join.setOnCondition(joinGen2.generateExpression()); + } + + JoinType newJoinType = TiDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation + // between LEFT and RIGHT + // join + newJoinType = Randomly.fromOptions(Arrays.stream(JoinType.values()) + .filter(j -> j != JoinType.LEFT && j != JoinType.RIGHT && j != JoinType.NATURAL) + .toArray(JoinType[]::new)); + } else { + newJoinType = Randomly.fromOptions(Arrays.stream(JoinType.values()) + .filter(j -> j != join.getJoinType() && j != JoinType.NATURAL).toArray(JoinType[]::new)); + } + + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + if (newJoinType == JoinType.CROSS) { + join.setOnCondition(null); + } + return increase; + } + + private boolean mutateWhere() { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(gen.generateExpression()); + } + return increase; + } + + private boolean mutateGroupBy() { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + select.clearHavingClause(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + private boolean mutateHaving() { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(gen.generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(gen.generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + private boolean mutateAnd() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression()); + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), gen.generateExpression(), + TiDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + private boolean mutateOr() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression()); + return false; + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), gen.generateExpression(), + TiDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + private boolean mutateLimit() { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(gen.generateConstant(TiDBDataType.INT)); + } + return increase; + } + + private double getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + throws AssertionError, SQLException { + double row = -1; + String explainQuery = "EXPLAIN " + selectStr; + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + double estRows = Double.parseDouble(rs.getString(2)); + if (row == -1) { + row = estRows; + } + String operation = rs.getString(1).split("_")[0]; // Extract operation names for query plans + queryPlanSequences.add(operation); + return estRows; + } + } + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + if (row == -1) { + throw new IgnoreMeException(); + } + return row; + } +} From 5208d661fdf00c3ac3a2a254a0ce05cc1b37ad68 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Fri, 23 Jun 2023 18:50:22 +0800 Subject: [PATCH 358/856] mysql: optimize code for CERT (#835) * mysql: remove dead code * mysql: add interfaces for CERT * mysql: code formatting --- src/sqlancer/mysql/MySQLProvider.java | 14 +++++++++ src/sqlancer/mysql/ast/MySQLSelect.java | 4 +++ .../mysql/gen/MySQLExpressionGenerator.java | 31 +------------------ .../mysql/gen/MySQLInsertGenerator.java | 13 +++++--- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 3 -- 5 files changed, 28 insertions(+), 37 deletions(-) diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 0f1704bf3..c1279f281 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -4,6 +4,8 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -18,6 +20,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; import sqlancer.mysql.gen.MySQLDeleteGenerator; import sqlancer.mysql.gen.MySQLDropIndex; @@ -188,4 +191,15 @@ public String getDBMSName() { return "mysql"; } + @Override + public boolean addRowsToAllTables(MySQLGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (MySQLTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = MySQLInsertGenerator.insertRow(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index 7abd1f639..3ed6df405 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -18,6 +18,10 @@ public void setSelectType(SelectType fromOptions) { this.setFromOptions(fromOptions); } + public SelectType getSelectType() { + return fromOptions; + } + public SelectType getFromOptions() { return fromOptions; } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 7911a0f3e..c3d754f58 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -65,10 +65,6 @@ public MySQLExpression generateExpression(int depth) { case UNARY_PREFIX_OPERATION: MySQLExpression subExpr = generateExpression(depth + 1); MySQLUnaryPrefixOperator random = MySQLUnaryPrefixOperator.getRandom(); - if (random == MySQLUnaryPrefixOperator.MINUS) { - // workaround for https://bugs.mysql.com/bug.php?id=99122 - throw new IgnoreMeException(); - } return new MySQLUnaryPrefixOperation(subExpr, random); case UNARY_POSTFIX: return new MySQLUnaryPostfixOperation(generateExpression(depth + 1), @@ -156,34 +152,9 @@ public MySQLExpression generateConstant() { case STRING: /* Replace characters that still trigger open bugs in MySQL */ String string = state.getRandomly().getString().replace("\\", "").replace("\n", ""); - if (string.startsWith("\n")) { - // workaround for https://bugs.mysql.com/bug.php?id=99130 - throw new IgnoreMeException(); - } - if (string.startsWith("-0") || string.startsWith("0.") || string.startsWith(".")) { - // https://bugs.mysql.com/bug.php?id=99145 - throw new IgnoreMeException(); - } - MySQLConstant createStringConstant = MySQLConstant.createStringConstant(string); - // if (Randomly.getBoolean()) { - // return new MySQLCollate(createStringConstant, - // Randomly.fromOptions("ascii_bin", "binary")); - // } - if (string.startsWith("1e")) { - // https://bugs.mysql.com/bug.php?id=99146 - throw new IgnoreMeException(); - } - return createStringConstant; + return MySQLConstant.createStringConstant(string); case DOUBLE: double val = state.getRandomly().getDouble(); - if (Math.abs(val) <= 1 && val != 0) { - // https://bugs.mysql.com/bug.php?id=99145 - throw new IgnoreMeException(); - } - if (Math.abs(val) > 1.0E30) { - // https://bugs.mysql.com/bug.php?id=99146 - throw new IgnoreMeException(); - } return new MySQLDoubleConstant(val); default: throw new AssertionError(); diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 496600905..14994a140 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -19,16 +19,21 @@ public class MySQLInsertGenerator { private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; - public MySQLInsertGenerator(MySQLGlobalState globalState) { + public MySQLInsertGenerator(MySQLGlobalState globalState, MySQLTable table) { this.globalState = globalState; - table = globalState.getSchema().getRandomTable(); + this.table = table; } public static SQLQueryAdapter insertRow(MySQLGlobalState globalState) throws SQLException { + MySQLTable table = globalState.getSchema().getRandomTable(); + return insertRow(globalState, table); + } + + public static SQLQueryAdapter insertRow(MySQLGlobalState globalState, MySQLTable table) throws SQLException { if (Randomly.getBoolean()) { - return new MySQLInsertGenerator(globalState).generateInsert(); + return new MySQLInsertGenerator(globalState, table).generateInsert(); } else { - return new MySQLInsertGenerator(globalState).generateReplace(); + return new MySQLInsertGenerator(globalState, table).generateReplace(); } } diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 00a477714..546962fe6 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -66,9 +66,6 @@ private enum Action { MAX_SP_RECURSION_DEPTH("max_sp_recursion_depth", (r) -> r.getLong(0, 255), Scope.GLOBAL, Scope.SESSION), // MYISAM_DATA_POINTER_SIZE("myisam_data_pointer_size", (r) -> r.getLong(2, 7), Scope.GLOBAL), // MYISAM_MAX_SORT_FILE_SIZE("myisam_max_sort_file_size", (r) -> r.getLong(0, 9223372036854775807L), Scope.GLOBAL), // - // MYISAM_REPAIR_THREADS("myisam_repair_threads", (r) -> r.getLong(1, Long.MAX_VALUE), Scope.GLOBAL, - // Scope.SESSION), // comment out this to avoid java.sql.SQLException: Unknown system variable - // 'myisam_repair_threads' MYISAM_SORT_BUFFER_SIZE("myisam_sort_buffer_size", (r) -> r.getLong(4096, Long.MAX_VALUE), Scope.GLOBAL, Scope.SESSION), // MYISAM_STATS_METHOD("myisam_stats_method", From 1f80de16e38c4a144e8dabbcef00d07cd15470f9 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Fri, 23 Jun 2023 21:50:23 +0800 Subject: [PATCH 359/856] Refactor: improve debugging experience (#819) * Update dependabot.yml * refactor: refactor ComparatorHelper.java and improve debugging experience * style: run code format * fix: manually drop a previous useless commit * refactor: keep minial change * refactor: keep minial change * refactor: different logging and assertion message * refactor: different logging and assertion message * naming: change log message --- src/sqlancer/ComparatorHelper.java | 35 +++++++++++++++++++----------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index c2fc009ef..41d2cf996 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -60,7 +60,7 @@ public static List getResultSetFirstColumnAsString(String queryString, E String resultTemp = result.getString(1); if (resultTemp != null) { resultTemp = resultTemp.replaceAll("[\\.]0+$", ""); // Remove the trailing zeros as many DBMS treat - // it as non-bugs + // it as non-bugs } resultSet.add(resultTemp); } @@ -87,13 +87,19 @@ public static List getResultSetFirstColumnAsString(String queryString, E public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, String originalQueryString, List combinedString, SQLGlobalState state) { if (resultSet.size() != secondResultSet.size()) { - String queryFormatString = "-- %s;\n-- cardinality: %d"; + String queryFormatString = "-- %s;" + System.lineSeparator() + "-- cardinality: %d" + + System.lineSeparator(); String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); - String secondQueryString = String.format(queryFormatString, - combinedString.stream().collect(Collectors.joining(";")), secondResultSet.size()); - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", - resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); + String combinedQueryString = String.join(";", combinedString); + String secondQueryString = String.format(queryFormatString, combinedQueryString, secondResultSet.size()); + state.getState().getLocalState() + .log(String.format("%s" + System.lineSeparator() + "%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format( + "The size of the result sets mismatch (%d and %d)!" + System.lineSeparator() + + "First query: \"%s\", whose cardinality is: %d" + System.lineSeparator() + + "Second query:\"%s\", whose cardinality is: %d", + resultSet.size(), secondResultSet.size(), originalQueryString, resultSet.size(), + combinedQueryString, secondResultSet.size()); throw new AssertionError(assertionMessage); } @@ -105,14 +111,17 @@ public static void assumeResultSetsAreEqual(List resultSet, List firstResultSetMisses.removeAll(secondHashSet); Set secondResultSetMisses = new HashSet<>(secondHashSet); secondResultSetMisses.removeAll(firstHashSet); - String queryFormatString = "-- %s;\n-- misses: %s"; + + String queryFormatString = "-- Query: \"%s\"; It misses: \"%s\""; String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); - String secondQueryString = String.format(queryFormatString, - combinedString.stream().collect(Collectors.joining(";")), secondResultSetMisses); + String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), + secondResultSetMisses); // update the SELECT queries to be logged at the bottom of the error log file - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", - firstQueryString, secondQueryString); + state.getState().getLocalState() + .log(String.format("%s" + System.lineSeparator() + "%s", firstQueryString, secondQueryString)); + String assertionMessage = String.format("The content of the result sets mismatch!" + System.lineSeparator() + + "First query : \"%s\"" + System.lineSeparator() + "Second query: \"%s\"", originalQueryString, + secondQueryString); throw new AssertionError(assertionMessage); } } From 3d37edc805dfa41454ee2a3d5e0ccd3effc298a7 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 26 Jun 2023 08:56:25 +0800 Subject: [PATCH 360/856] mysql: cert init (#837) * mysql: cert init * CERT oracle base class * mysql: stream API optimization * cert: abstract mutate method * cert: getRandomExcept --- .../common/oracle/CERTOracleBase.java | 88 +++++++ src/sqlancer/mysql/MySQLOptions.java | 14 +- src/sqlancer/mysql/MySQLProvider.java | 21 ++ src/sqlancer/mysql/ast/MySQLSelect.java | 4 - .../mysql/oracle/MySQLCERTOracle.java | 217 ++++++++++++++++++ src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 72 ++---- 6 files changed, 359 insertions(+), 57 deletions(-) create mode 100644 src/sqlancer/common/oracle/CERTOracleBase.java create mode 100644 src/sqlancer/mysql/oracle/MySQLCERTOracle.java diff --git a/src/sqlancer/common/oracle/CERTOracleBase.java b/src/sqlancer/common/oracle/CERTOracleBase.java new file mode 100644 index 000000000..42e8e5833 --- /dev/null +++ b/src/sqlancer/common/oracle/CERTOracleBase.java @@ -0,0 +1,88 @@ +package sqlancer.common.oracle; + +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; + +public abstract class CERTOracleBase> implements TestOracle { + + protected final S state; + protected final ExpectedErrors errors; + protected List queryPlan1Sequences; + protected List queryPlan2Sequences; + + protected enum Mutator { + JOIN, DISTINCT, WHERE, GROUPBY, HAVING, AND, OR, LIMIT; + + public static Mutator getRandomExcept(Mutator... exclude) { + Mutator[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(Mutator[]::new); + return Randomly.fromOptions(values); + } + } + + protected CERTOracleBase(S state) { + this.state = state; + this.errors = new ExpectedErrors(); + } + + protected boolean mutate(Mutator... exclude) { + Mutator m = Mutator.getRandomExcept(exclude); + switch (m) { + case JOIN: + return mutateJoin(); + case DISTINCT: + return mutateDistinct(); + case WHERE: + return mutateWhere(); + case GROUPBY: + return mutateGroupBy(); + case HAVING: + return mutateHaving(); + case AND: + return mutateAnd(); + case OR: + return mutateOr(); + case LIMIT: + return mutateLimit(); + default: + throw new AssertionError(m); + } + } + + protected boolean mutateJoin() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateDistinct() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateWhere() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateGroupBy() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateHaving() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateAnd() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateOr() { + throw new UnsupportedOperationException(); + } + + protected boolean mutateLimit() { + throw new UnsupportedOperationException(); + } + +} diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index ad8060090..2d9e80bc0 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -11,6 +11,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; +import sqlancer.mysql.oracle.MySQLCERTOracle; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @@ -45,7 +46,18 @@ public boolean requiresAllTablesToContainRows() { return true; } - } + }, + CERT { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }; } @Override diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index c1279f281..0c3a1f2e7 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -18,8 +18,10 @@ import sqlancer.SQLProviderAdapter; import sqlancer.StatementExecutor; import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; import sqlancer.mysql.gen.MySQLDeleteGenerator; @@ -36,6 +38,7 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLChecksum; import sqlancer.mysql.gen.tblmaintenance.MySQLOptimize; import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; +import sqlancer.mysql.oracle.MySQLCERTOracle; @AutoService(DatabaseProvider.class) public class MySQLProvider extends SQLProviderAdapter { @@ -153,6 +156,24 @@ public void generateDatabase(MySQLGlobalState globalState) throws Exception { } }); se.executeStatements(); + + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 + && globalState.getDbmsSpecificOptions().getTestOracleFactory().get(0) + .create(globalState) instanceof MySQLCERTOracle) { + // Enfore statistic collected for all tables + ExpectedErrors errors = new ExpectedErrors(); + MySQLErrors.addExpressionErrors(errors); + for (MySQLTable table : globalState.getSchema().getDatabaseTables()) { + StringBuilder sb = new StringBuilder(); + sb.append("ANALYZE TABLE "); + sb.append(table.getName()); + sb.append(" UPDATE HISTOGRAM ON "); + String columns = table.getColumns().stream().map(MySQLColumn::getName) + .collect(Collectors.joining(", ")); + sb.append(columns + ";"); + globalState.executeStatement(new SQLQueryAdapter(sb.toString(), errors)); + } + } } @Override diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index 3ed6df405..7abd1f639 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -18,10 +18,6 @@ public void setSelectType(SelectType fromOptions) { this.setFromOptions(fromOptions); } - public SelectType getSelectType() { - return fromOptions; - } - public SelectType getFromOptions() { return fromOptions; } diff --git a/src/sqlancer/mysql/oracle/MySQLCERTOracle.java b/src/sqlancer/mysql/oracle/MySQLCERTOracle.java new file mode 100644 index 000000000..2cdf3c7cf --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLCERTOracle.java @@ -0,0 +1,217 @@ +package sqlancer.mysql.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CERTOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + +public class MySQLCERTOracle extends CERTOracleBase implements TestOracle { + private MySQLExpressionGenerator gen; + private MySQLSelect select; + + public MySQLCERTOracle(MySQLGlobalState globalState) { + super(globalState); + MySQLErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + queryPlan1Sequences = new ArrayList<>(); + queryPlan2Sequences = new ArrayList<>(); + + // Randomly generate a query + MySQLTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new MySQLExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList())); + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) + .collect(Collectors.toList()); + + select = new MySQLSelect(); + select.setFetchColumns(fetchColumns); + select.setFromList(tableList); + + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateExpression()); + } + } + + // Set the join. Todo: to make it random + // List joinExpressions = getJoins(tableList, state); + // select.setJoinList(joinExpressions); + + // Get the result of the first query + String queryString1 = MySQLVisitor.asString(select); + int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + boolean increase = mutate(Mutator.JOIN, Mutator.LIMIT); + + // Get the result of the second query + String queryString2 = MySQLVisitor.asString(select); + int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + // Check structural equivalence + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check the results + if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { + throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 + + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + } + } + + @Override + protected boolean mutateDistinct() { + MySQLSelect.SelectType selectType = select.getFromOptions(); + if (selectType != MySQLSelect.SelectType.ALL) { + select.setSelectType(MySQLSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(MySQLSelect.SelectType.DISTINCT); + return false; + } + } + + @Override + protected boolean mutateWhere() { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(gen.generateExpression()); + } + return increase; + } + + @Override + protected boolean mutateGroupBy() { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + @Override + protected boolean mutateHaving() { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(gen.generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(gen.generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + @Override + protected boolean mutateAnd() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression()); + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(), MySQLBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + @Override + protected boolean mutateOr() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression()); + return false; + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(), MySQLBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + // The limit clause only accpets positive integers, which is not supported yet + // private boolean mutateLimit() { + // boolean increase = select.getLimitClause() != null; + // if (increase) { + // select.setLimitClause(null); + // } else { + // select.setLimitClause(gen.generateConstant()); + // } + // return increase; + // } + + private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + throws AssertionError, SQLException { + int row = -1; + String explainQuery = "EXPLAIN " + selectStr; + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + int estRows = rs.getInt(10); + if (row == -1) { + row = estRows; + } + String operation = rs.getString(2); + queryPlanSequences.add(operation); + } + } + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + if (row == -1) { + throw new IgnoreMeException(); + } + return row; + } + +} diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java index d23789b26..1bef4ac96 100644 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -11,8 +11,8 @@ import sqlancer.Randomly; import sqlancer.SQLGlobalState; import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CERTOracleBase; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.tidb.TiDBErrors; @@ -31,24 +31,12 @@ import sqlancer.tidb.ast.TiDBTableReference; import sqlancer.tidb.visitor.TiDBVisitor; -public class TiDBCERTOracle implements TestOracle { - private final TiDBGlobalState state; +public class TiDBCERTOracle extends CERTOracleBase implements TestOracle { private TiDBExpressionGenerator gen; - private final ExpectedErrors errors = new ExpectedErrors(); private TiDBSelect select; - private List queryPlan1Sequences; - private List queryPlan2Sequences; - - public enum Mutator { - JOIN, WHERE, GROUPBY, HAVING, AND, OR, LIMIT; - - public static Mutator getRandom() { - return Randomly.fromOptions(values()); - } - } public TiDBCERTOracle(TiDBGlobalState globalState) { - state = globalState; + super(globalState); TiDBErrors.addExpressionErrors(errors); } @@ -96,34 +84,7 @@ public void check() throws SQLException { double rowCount1 = getRow(state, queryString1, queryPlan1Sequences); // Mutate the query - boolean increase = false; - - Mutator mutation = Mutator.getRandom(); - switch (mutation) { - case JOIN: - increase = mutateJoin(); - break; - case WHERE: - increase = mutateWhere(); - break; - case GROUPBY: - increase = mutateGroupBy(); - break; - case HAVING: - increase = mutateHaving(); - break; - case AND: - increase = mutateAnd(); - break; - case OR: - increase = mutateOr(); - break; - case LIMIT: - increase = mutateLimit(); - break; - default: - throw new AssertionError(); - } + boolean increase = mutate(Mutator.DISTINCT); // Get the result of the second query String queryString2 = TiDBVisitor.asString(select); @@ -142,12 +103,13 @@ public void check() throws SQLException { */ // Check the results if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { - throw new AssertionError("Mutator: " + mutation + ", Inconsistent result for query: EXPLAIN " + queryString1 - + "; --" + rowCount1 + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 + + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); } } - private boolean mutateJoin() { + @Override + protected boolean mutateJoin() { if (select.getJoinList().isEmpty()) { return false; } @@ -183,7 +145,8 @@ private boolean mutateJoin() { return increase; } - private boolean mutateWhere() { + @Override + protected boolean mutateWhere() { boolean increase = select.getWhereClause() != null; if (increase) { select.setWhereClause(null); @@ -193,7 +156,8 @@ private boolean mutateWhere() { return increase; } - private boolean mutateGroupBy() { + @Override + protected boolean mutateGroupBy() { boolean increase = select.getGroupByExpressions().size() > 0; if (increase) { select.clearGroupByExpressions(); @@ -204,7 +168,8 @@ private boolean mutateGroupBy() { return increase; } - private boolean mutateHaving() { + @Override + protected boolean mutateHaving() { if (select.getGroupByExpressions().size() == 0) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(gen.generateExpression()); @@ -220,7 +185,8 @@ private boolean mutateHaving() { } } - private boolean mutateAnd() { + @Override + protected boolean mutateAnd() { if (select.getWhereClause() == null) { select.setWhereClause(gen.generateExpression()); } else { @@ -231,7 +197,8 @@ private boolean mutateAnd() { return false; } - private boolean mutateOr() { + @Override + protected boolean mutateOr() { if (select.getWhereClause() == null) { select.setWhereClause(gen.generateExpression()); return false; @@ -243,7 +210,8 @@ private boolean mutateOr() { } } - private boolean mutateLimit() { + @Override + protected boolean mutateLimit() { boolean increase = select.getLimitClause() != null; if (increase) { select.setLimitClause(null); From 6957c891beb1f5fec38d1f0db18d752ba47d8697 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 26 Jun 2023 08:58:23 +0800 Subject: [PATCH 361/856] feat: stonedb follow up implementation (#836) * feat: add init support feat: add support feat: init support of stonedb * feat: add init support * fix: optimize imports * fix: change from MySQL to StoneDB options * fix: connection string * fix: StoneDBSchema.java * fix: StoneDBSchema.java unused enum * feat: add StoneDB table generator * feat: add support for Action and mapActions * feat: add appendType StoneDBTableGenerator.java * fix: fix errors in mvn verify -DskipTests * feat: add expression generator for StoneDB * feat: add alter index insert generator * refactor: refactor according to comments, * fix: naming error and mvn verify error * fix: simplify the BigInteger generation * fix: fix the BigInteger generation * feat: add drop index and table update * style: add comments * feat: let StoneDBInsertGenerator extends AbstractInsertGenerator --- src/check_names.py | 1 + src/sqlancer/Randomly.java | 14 +- src/sqlancer/stonedb/StoneDBBugs.java | 6 + src/sqlancer/stonedb/StoneDBErrors.java | 7 + src/sqlancer/stonedb/StoneDBOptions.java | 46 +++ src/sqlancer/stonedb/StoneDBProvider.java | 122 ++++++ src/sqlancer/stonedb/StoneDBSchema.java | 342 +++++++++++++++++ .../stonedb/StoneDBToStringVisitor.java | 27 ++ src/sqlancer/stonedb/ast/StoneDBConstant.java | 206 ++++++++++ .../stonedb/ast/StoneDBExpression.java | 4 + .../stonedb/gen/StoneDBDeleteGenerator.java | 60 +++ .../gen/StoneDBExpressionGenerator.java | 287 ++++++++++++++ .../gen/StoneDBIndexDropGenerator.java | 47 +++ .../stonedb/gen/StoneDBIndexGenerator.java | 75 ++++ .../stonedb/gen/StoneDBInsertGenerator.java | 99 +++++ .../gen/StoneDBTableAlterGenerator.java | 94 +++++ .../gen/StoneDBTableCreateGenerator.java | 357 ++++++++++++++++++ .../gen/StoneDBTableUpdateGenerator.java | 69 ++++ .../stonedb/oracle/StoneDBNoRECOracle.java | 16 + .../stonedb/oracle/StoneDBTLPOracle.java | 16 + 20 files changed, 1893 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/stonedb/StoneDBBugs.java create mode 100644 src/sqlancer/stonedb/StoneDBErrors.java create mode 100644 src/sqlancer/stonedb/StoneDBOptions.java create mode 100644 src/sqlancer/stonedb/StoneDBProvider.java create mode 100644 src/sqlancer/stonedb/StoneDBSchema.java create mode 100644 src/sqlancer/stonedb/StoneDBToStringVisitor.java create mode 100644 src/sqlancer/stonedb/ast/StoneDBConstant.java create mode 100644 src/sqlancer/stonedb/ast/StoneDBExpression.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java create mode 100644 src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java diff --git a/src/check_names.py b/src/check_names.py index 1b110370b..a88c94e89 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -53,4 +53,5 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes name_to_files["Doris"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "doris")) + name_to_files["StoneDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "stonedb")) verify_all_dbs(name_to_files) diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index e1250abb3..2ceacfb30 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -1,6 +1,7 @@ package sqlancer; import java.math.BigDecimal; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -215,7 +216,6 @@ public String getString(Randomly r) { }, ALPHANUMERIC { - @Override public String getString(Randomly r) { return getStringOfAlphabet(r, ALPHANUMERIC_ALPHABET); @@ -224,7 +224,6 @@ public String getString(Randomly r) { }, ALPHANUMERIC_SPECIALCHAR { - @Override public String getString(Randomly r) { return getStringOfAlphabet(r, ALPHANUMERIC_SPECIALCHAR_ALPHABET); @@ -452,6 +451,17 @@ public long getLong(long left, long right) { return getNextLong(left, right); } + public BigInteger getBigInteger(BigInteger left, BigInteger right) { + if (left.equals(right)) { + return left; + } + BigInteger result = new BigInteger(String.valueOf(getInteger(left.intValue(), right.intValue()))); + if (result.compareTo(left) < 0 && result.compareTo(right) > 0) { + throw new IgnoreMeException(); + } + return result; + } + public BigDecimal getRandomBigDecimal() { return BigDecimal.valueOf(getThreadRandom().get().nextDouble()); } diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java new file mode 100644 index 000000000..7ed539763 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBBugs.java @@ -0,0 +1,6 @@ +package sqlancer.stonedb; + +public final class StoneDBBugs { + private StoneDBBugs() { + } +} diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java new file mode 100644 index 000000000..54fd28ad8 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -0,0 +1,7 @@ +package sqlancer.stonedb; + +public final class StoneDBErrors { + private StoneDBErrors() { + } + +} diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java new file mode 100644 index 000000000..928bcfa39 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBOptions.java @@ -0,0 +1,46 @@ +package sqlancer.stonedb; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.stonedb.StoneDBOptions.StoneDBOracleFactory; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.oracle.StoneDBNoRECOracle; +import sqlancer.stonedb.oracle.StoneDBTLPOracle; + +@Parameters(separators = "=", commandDescription = "StoneDB (default host: " + StoneDBOptions.DEFAULT_HOST + + ", default port: " + StoneDBOptions.DEFAULT_PORT + ")") +public class StoneDBOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 3306; + + @Parameter(names = "--oracle") + public List oracles = Arrays.asList(StoneDBOracleFactory.NOREC); + + public enum StoneDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBNoRECOracle(globalState); + } + }, + TLP { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws Exception { + return new StoneDBTLPOracle(globalState); + } + } + } + + @Override + public List getTestOracleFactory() { + return oracles; + } +} diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java new file mode 100644 index 000000000..376235d92 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBProvider.java @@ -0,0 +1,122 @@ +package sqlancer.stonedb; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.stonedb.gen.StoneDBDeleteGenerator; +import sqlancer.stonedb.gen.StoneDBIndexGenerator; +import sqlancer.stonedb.gen.StoneDBInsertGenerator; +import sqlancer.stonedb.gen.StoneDBTableAlterGenerator; +import sqlancer.stonedb.gen.StoneDBTableCreateGenerator; + +@AutoService(DatabaseProvider.class) +public class StoneDBProvider extends SQLProviderAdapter { + + public StoneDBProvider() { + super(StoneDBGlobalState.class, StoneDBOptions.class); + } + + public static class StoneDBGlobalState extends SQLGlobalState { + @Override + protected StoneDBSchema readSchema() throws Exception { + return StoneDBSchema.fromConnection(getConnection(), getDatabaseName()); + } + } + + enum Action implements AbstractAction { + SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), // + ALTER_TABLE(StoneDBTableAlterGenerator::generate), // + DELETE(StoneDBDeleteGenerator::generate), // + INDEX(StoneDBIndexGenerator::generate), // + INSERT(StoneDBInsertGenerator::generate); // + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public Query getQuery(StoneDBGlobalState globalState) throws Exception { + return sqlQueryProvider.getQuery(globalState); + } + } + + private static int mapActions(StoneDBGlobalState globalState, Action a) { + globalState.getRandomly(); + switch (a) { + case SHOW_TABLES: + return 1; + default: + throw new AssertionError(a); + } + } + + @Override + public void generateDatabase(StoneDBGlobalState globalState) throws Exception { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + SQLQueryAdapter createTable = StoneDBTableCreateGenerator.generate(globalState, tableName); + globalState.executeStatement(createTable); + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + StoneDBProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(StoneDBGlobalState globalState) throws Exception { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = StoneDBOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = StoneDBOptions.DEFAULT_PORT; + } + String databaseName = globalState.getDatabaseName(); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", + host, port); + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "stonedb"; + } +} diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java new file mode 100644 index 000000000..eccf05c27 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -0,0 +1,342 @@ +package sqlancer.stonedb; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.stonedb.ast.StoneDBConstant; +import sqlancer.stonedb.ast.StoneDBExpression; + +public class StoneDBSchema extends AbstractSchema { + + public enum StoneDBDataType { + + NULL, TINYINT, SMALLINT, MEDIUMINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, YEAR, TIME, DATE, DATETIME, TIMESTAMP, + CHAR, VARCHAR, TINYTEXT, TEXT, MEDIUMTEXT, LONGTEXT, BINARY, VARBINARY, TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB; + + public static StoneDBDataType getRandomWithoutNull() { + return Randomly.fromOptions(values()); + } + + public static Node getRandomValue(StoneDBDataType dataType) { + switch (dataType) { + case TINYINT: + break; + case SMALLINT: + break; + case MEDIUMINT: + break; + case INT: + return StoneDBConstant.createIntConstant((int) Randomly.getNonCachedInteger()); + case BIGINT: + break; + case FLOAT: + break; + case DOUBLE: + return StoneDBConstant.createDoubleConstant(Randomly.getUncachedDouble()); + case DECIMAL: + break; + case YEAR: + break; + case TIME: + break; + case DATE: + return StoneDBConstant.createDateConstant(Randomly.getNonCachedInteger()); + case DATETIME: + break; + case TIMESTAMP: + return StoneDBConstant.createTimestampConstant(Randomly.getNonCachedInteger()); + case CHAR: + break; + case VARCHAR: + break; + case TINYTEXT: + break; + case TEXT: + return StoneDBConstant.createTextConstant(new Randomly().getString()); + case MEDIUMTEXT: + break; + case LONGTEXT: + break; + case BINARY: + break; + case VARBINARY: + break; + case TINYBLOB: + break; + case BLOB: + break; + case MEDIUMBLOB: + break; + case LONGBLOB: + break; + default: + throw new AssertionError(); + } + return null; + } + } + + public static class StoneDBTable + extends AbstractRelationalTable { + + public StoneDBTable(String tableName, List columns, List indexes, boolean isView) { + super(tableName, columns, indexes, isView); + } + + public boolean hasPrimaryKey() { + return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); + } + + } + + public static final class StoneDBIndex extends TableIndex { + private StoneDBIndex(String indexName) { + super(indexName); + } + + public static StoneDBIndex create(String indexName) { + return new StoneDBIndex(indexName); + } + + @Override + public String getIndexName() { + if (super.getIndexName().contentEquals("PRIMARY")) { + return "`PRIMARY`"; + } else { + return super.getIndexName(); + } + } + } + + public static StoneDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con, databaseName); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, databaseName, tableName); + List indexes = getIndexes(con, databaseName, tableName); + boolean isView = tableName.startsWith("v"); + StoneDBTable t = new StoneDBTable(tableName, databaseColumns, indexes, isView); + for (StoneDBColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + + } + return new StoneDBSchema(databaseTables); + } + + private static List getIndexes(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List indexes = new ArrayList<>(); + try (ResultSet rs = con.createStatement() + .executeQuery("SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = '" + + databaseName + "' AND TABLE_NAME='" + tableName + "';")) { + while (rs.next()) { + String indexName = rs.getString("INDEX_NAME"); + indexes.add(StoneDBIndex.create(indexName)); + } + } + return indexes; + } + + private static List getTableColumns(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List columns = new ArrayList<>(); + try (ResultSet rs = con.createStatement() + .executeQuery("select * from information_schema.columns where table_schema = '" + databaseName + + "' AND TABLE_NAME='" + tableName + "';")) { + while (rs.next()) { + String columnName = rs.getString("COLUMN_NAME"); + String dataType = rs.getString("DATA_TYPE"); + int precision = rs.getInt("NUMERIC_PRECISION"); + boolean isNullable = !rs.getString("IS_NULLABLE").equals("NO"); + boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); + StoneDBColumn c = new StoneDBColumn(columnName, getColumnCompositeDataType(dataType), isPrimaryKey, + isNullable, precision); + columns.add(c); + } + } + return columns; + } + + private static StoneDBCompositeDataType getColumnCompositeDataType(String typeString) { + if (Arrays.stream(StoneDBDataType.values()).noneMatch(e -> e.name().equals(typeString.toUpperCase()))) { + throw new AssertionError(typeString); + } + return new StoneDBCompositeDataType(StoneDBDataType.valueOf(typeString.toUpperCase())); + } + + public static StoneDBDataType getColumnDataType(String typeString) { + if (Arrays.stream(StoneDBDataType.values()).noneMatch(e -> e.name().equals(typeString.toUpperCase()))) { + throw new AssertionError(typeString); + } + return StoneDBDataType.valueOf(typeString.toUpperCase()); + } + + private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { + List tableNames = new ArrayList<>(); + try (ResultSet rs = con.createStatement() + .executeQuery("select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + + databaseName + "';")) { + while (rs.next()) { + if (rs.getString("ENGINE").equals("TIANMU")) { + tableNames.add(rs.getString("TABLE_NAME")); + } + } + } + return tableNames; + } + + public static class StoneDBColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + private final int precision; + + public StoneDBColumn(String name, StoneDBCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable, + int precision) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + this.precision = precision; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + public int getPrecision() { + return precision; + } + + } + + public StoneDBSchema(List databaseTables) { + super(databaseTables); + } + + public static class StoneDBCompositeDataType { + private final StoneDBDataType dataType; + private final int size; + + public StoneDBCompositeDataType(StoneDBDataType dataType, int size) { + this.dataType = dataType; + this.size = size; + } + + public StoneDBCompositeDataType(StoneDBDataType dataType) { + this.dataType = dataType; + int size = -1; + switch (dataType) { + case TINYINT: + size = 1; + break; + case SMALLINT: + size = 2; + break; + case MEDIUMINT: + size = 3; + break; + case INT: + size = 4; + break; + case BIGINT: + size = 8; + break; + case FLOAT: + size = 4; + break; + case DOUBLE: + size = 8; + break; + case DECIMAL: + size = -2; + break; + case YEAR: + size = -2; + break; + case TIME: + size = -2; + break; + case DATE: + size = -2; + break; + case DATETIME: + size = -2; + break; + case TIMESTAMP: + size = -2; + break; + case CHAR: + size = -2; + break; + case VARCHAR: + size = -2; + break; + case TINYTEXT: + size = -2; + break; + case TEXT: + size = -2; + break; + case MEDIUMTEXT: + size = -2; + break; + case LONGTEXT: + size = -2; + break; + case BINARY: + size = -2; + break; + case VARBINARY: + size = -2; + break; + case TINYBLOB: + size = -2; + break; + case BLOB: + size = -2; + break; + case MEDIUMBLOB: + size = -2; + break; + case LONGBLOB: + size = -2; + break; + default: + throw new AssertionError(); + } + this.size = size; + } + + public StoneDBDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public static StoneDBCompositeDataType getRandomWithoutNull() { + StoneDBDataType type = StoneDBDataType.getRandomWithoutNull(); + return new StoneDBCompositeDataType(type); + } + } +} diff --git a/src/sqlancer/stonedb/StoneDBToStringVisitor.java b/src/sqlancer/stonedb/StoneDBToStringVisitor.java new file mode 100644 index 000000000..3cebd97f1 --- /dev/null +++ b/src/sqlancer/stonedb/StoneDBToStringVisitor.java @@ -0,0 +1,27 @@ +package sqlancer.stonedb; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.stonedb.ast.StoneDBConstant; +import sqlancer.stonedb.ast.StoneDBExpression; + +public class StoneDBToStringVisitor extends NewToStringVisitor { + @Override + public void visitSpecific(Node expr) { + if (expr instanceof StoneDBConstant) { + visit((StoneDBConstant) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(StoneDBConstant constant) { + sb.append(constant.toString()); + } + + public static String asString(Node expr) { + StoneDBToStringVisitor visitor = new StoneDBToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/stonedb/ast/StoneDBConstant.java b/src/sqlancer/stonedb/ast/StoneDBConstant.java new file mode 100644 index 000000000..f0abae48a --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBConstant.java @@ -0,0 +1,206 @@ +package sqlancer.stonedb.ast; + +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +import sqlancer.common.ast.newast.Node; + +public class StoneDBConstant implements Node { + + private StoneDBConstant() { + } + + public static class StoneDBNullConstant extends StoneDBConstant { + @Override + public String toString() { + return "NULL"; + } + } + + public static Node createNullConstant() { + return new StoneDBNullConstant(); + } + + public static class StoneDBIntConstant extends StoneDBConstant { + private final Integer value; + + public StoneDBIntConstant(int value) { + this.value = value; + } + + @Override + public String toString() { + if (value.equals(Integer.MIN_VALUE)) { + return "INT_NULL"; + } + return String.valueOf(value); + } + } + + public static Node createIntConstant(int val) { + return new StoneDBIntConstant(val); + } + + public static class StoneDBBigIntConstant extends StoneDBConstant { + private final Long value; + + public StoneDBBigIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + // For INT type: -2147483648 is reserved to indicate INT_NULL in Tianmu engine, Minimum Value Signed start + // from -2147483647 + // refer: https://stonedb.io/docs/SQL-reference/data-types + if (value.equals(Long.MIN_VALUE)) { + return "BIGINT_NULL"; + } + return String.valueOf(value); + } + } + + public static class StoneDBDoubleConstant extends StoneDBConstant { + + private final Double value; + + public StoneDBDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + + } + + public static Node createDoubleConstant(double val) { + return new StoneDBDoubleConstant(val); + } + + public static class StoneDBTextConstant extends StoneDBConstant { + + private final String value; + + public StoneDBTextConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static Node createTextConstant(String text) { + return new StoneDBTextConstant(text); + } + + public static class StoneDBBitConstant extends StoneDBConstant { + + private final String value; + + public StoneDBBitConstant(long value) { + this.value = Long.toBinaryString(value); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "B'" + value + "'"; + } + + } + + public static class StoneDBDateConstant extends StoneDBConstant { + + public String textRepr; + + public StoneDBDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + + } + + public static Node createDateConstant(long integer) { + return new StoneDBDateConstant(integer); + } + + public static class StoneDBTimestampConstant extends StoneDBConstant { + + public String textRepr; + + public StoneDBTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + + } + + public static Node createTimestampConstant(long integer) { + return new StoneDBTimestampConstant(integer); + } + + public static class StoneDBBooleanConstant extends StoneDBConstant { + + private final boolean value; + + public StoneDBBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + } + + public static Node createBooleanConstant(boolean val) { + return new StoneDBBooleanConstant(val); + } + +} diff --git a/src/sqlancer/stonedb/ast/StoneDBExpression.java b/src/sqlancer/stonedb/ast/StoneDBExpression.java new file mode 100644 index 000000000..fb8e75124 --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBExpression.java @@ -0,0 +1,4 @@ +package sqlancer.stonedb.ast; + +public interface StoneDBExpression { +} diff --git a/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java new file mode 100644 index 000000000..d52aed526 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java @@ -0,0 +1,60 @@ +package sqlancer.stonedb.gen; + +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public final class StoneDBDeleteGenerator { + private final StoneDBGlobalState globalState; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + private StoneDBDeleteGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBDeleteGenerator(globalState).getQuery(); + } + + public SQLQueryAdapter getQuery() { + StoneDBTable randomTable = globalState.getSchema().getRandomTable(); + sb.append("DELETE"); + if (Randomly.getBoolean()) { + sb.append(" LOW_PRIORITY"); + } + if (Randomly.getBoolean()) { + sb.append(" QUICK"); + } + if (Randomly.getBoolean()) { + sb.append(" IGNORE"); + } + sb.append(" FROM "); + sb.append(randomTable.getName()); + if (Randomly.getBoolean()) { + sb.append(" AS "); + sb.append(globalState.getSchema().getRandomTable().getName()); + } + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) + .setColumns(randomTable.getColumns()).generateExpression())); + } + if (Randomly.getBoolean()) { + sb.append(" ORDER BY "); + sb.append(Randomly.fromOptions( + randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))); + } + if (Randomly.getBoolean()) { + sb.append(" LIMIT "); + sb.append(new Randomly().getInteger()); + } + return new SQLQueryAdapter(sb.toString(), errors); + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java new file mode 100644 index 000000000..cf578ecb0 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -0,0 +1,287 @@ +package sqlancer.stonedb.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewCaseOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.ast.StoneDBConstant; +import sqlancer.stonedb.ast.StoneDBExpression; + +public class StoneDBExpressionGenerator extends UntypedExpressionGenerator, StoneDBColumn> { + + private final StoneDBGlobalState globalState; + + public StoneDBExpressionGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + } + + private enum Expression { + UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, BINARY_BITWISE, BETWEEN, IN, + CASE + } + + @Override + public Node negatePredicate(Node predicate) { + return null; + } + + @Override + public Node isNull(Node expr) { + return null; + } + + @Override + public Node generateConstant() { + if (Randomly.getBooleanWithSmallProbability()) { + return StoneDBConstant.createNullConstant(); + } + StoneDBDataType type = StoneDBDataType.getRandomWithoutNull(); + return generateConstant(type); + } + + public Node generateConstant(StoneDBDataType dataType) { + switch (dataType) { + case INT: + return StoneDBConstant + .createIntConstant(globalState.getRandomly().getInteger(Integer.MIN_VALUE + 1, Integer.MAX_VALUE)); + case DATE: + return StoneDBConstant.createDateConstant(globalState.getRandomly().getInteger()); + case TIMESTAMP: + return StoneDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case VARCHAR: + return StoneDBConstant.createTextConstant(globalState.getRandomly().getString()); + case DOUBLE: + return StoneDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + default: + throw new IgnoreMeException(); + } + } + + public Node generateConstant(StoneDBDataType dataType, boolean isNullable) { + if (isNullable && Randomly.getBooleanWithSmallProbability()) { + generateConstant(StoneDBDataType.NULL); + } + return generateConstant(dataType); + } + + @Override + protected Node generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBoolean()) { + StoneDBAggregateFunction aggregateFunction = StoneDBAggregateFunction.getRandom(); + allowAggregates = false; + return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs(), depth + 1), + aggregateFunction); + } + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + Operator op; + switch (expr) { + case UNARY_PREFIX: + op = StoneDBUnaryPrefixOperator.getRandom(); + return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), op); + case UNARY_POSTFIX: + op = StoneDBUnaryPostfixOperator.getRandom(); + return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), op); + case BINARY_COMPARISON: + op = StoneDBBinaryComparisonOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = StoneDBBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + op = StoneDBBinaryArithmeticOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_BITWISE: + op = StoneDBBinaryBitwiseOperator.getRandom(); + return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BETWEEN: + return new NewBetweenOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new NewInOperatorNode<>(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new NewCaseOperatorNode<>(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(); + } + } + + @Override + protected Node generateColumn() { + return null; + } + + public enum StoneDBAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1), FIRST(1), SUM(1); + + private int nrArgs; + + StoneDBAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static StoneDBAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + public enum StoneDBUnaryPrefixOperator implements Operator { + + NOT("NOT"), PLUS("+"), MINUS("-"); + + private final String textRepr; + + StoneDBUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static StoneDBUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum StoneDBUnaryPostfixOperator implements Operator { + + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private final String textRepr; + + StoneDBUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + public static StoneDBUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + } + + /* + * Comparison operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/comparison-operators + */ + public enum StoneDBBinaryComparisonOperator implements Operator { + EQUAL("="), GREATER(">"), LESS("<"), GREATER_EQUAL(">="), LESS_EQUAL("<="), + NOT_EQUALS(Randomly.fromList(Arrays.asList("!=", "<>"))), NULL_SAFE_EQUAL("<=>"), IN("IN"), NOT_IN("NOT_IN"), + LIKE("LIKE"), IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private final String textRepr; + + StoneDBBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + /* + * Logical operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/logical-operators + */ + public enum StoneDBBinaryLogicalOperator implements Operator { + + NOT("NOT"), AND("AND"), OR("OR"), XOR("XOR"); + + private final String textRepr; + + StoneDBBinaryLogicalOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + /* + * Arithmetic operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/arithmetic-operators + */ + public enum StoneDBBinaryArithmeticOperator implements Operator { + ADDITION("+"), MINUS("-"), MULTIPLICATION("*"), DIVISION(Randomly.fromList(Arrays.asList("/", " div "))), + MODULO(Randomly.fromList(Arrays.asList("%", " mod "))); + + private final String textRepr; + + StoneDBBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } + + /* + * Bitwise operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/bitwise-operators + */ + public enum StoneDBBinaryBitwiseOperator implements Operator { + AND("&"), OR("|"), XOR("^"), INVERSION("!"), LEFTSHIFT("<<"), RIGHTSHIFT(">>"); + + private final String textRepr; + + StoneDBBinaryBitwiseOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java new file mode 100644 index 000000000..dda72c735 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java @@ -0,0 +1,47 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; + +public class StoneDBIndexDropGenerator { + // which table to drop index + StoneDBSchema.StoneDBTable table; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBIndexDropGenerator(StoneDBGlobalState globalState) { + table = globalState.getSchema().getRandomTable(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBIndexDropGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + if (!table.hasIndexes()) { + return null; + } + sb.append("DROP INDEX "); + sb.append(table.getRandomIndex().getIndexName()); + sb.append(" ON "); + sb.append(table.getName()); + appendAlgoOrLockOption(); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private void appendAlgoOrLockOption() { + if (Randomly.getBoolean()) { + return; + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); + sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); + } else { + sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); + sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java new file mode 100644 index 000000000..9a86cf7e4 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java @@ -0,0 +1,75 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; + +public class StoneDBIndexGenerator { + private final StoneDBGlobalState globalState; + // which table to add index + StoneDBTable table; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBIndexGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + table = globalState.getSchema().getRandomTable(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBIndexGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("CREATE "); + sb.append(Randomly.fromOptions("UNIQUE", "FULLTEXT", "SPATIAL")); + sb.append(" INDEX"); + sb.append(globalState.getSchema().getFreeIndexName()); + appendIndexType(); + sb.append(" ON "); + sb.append(table.getName()); + appendKeyPart(); + appendIndexOption(); + appendAlgoOrLockOption(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void appendIndexType() { + if (Randomly.getBoolean()) { + return; + } + sb.append(" USING "); + sb.append(Randomly.fromOptions("BTREE", "HASH")); + } + + private void appendKeyPart() { + sb.append("("); + sb.append(table.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + sb.append(")"); + } + + private void appendIndexOption() { + if (Randomly.getBoolean()) { + return; + } + sb.append(Randomly.fromOptions(" VISIBLE", " INVISIBLE")); + } + + private void appendAlgoOrLockOption() { + if (Randomly.getBoolean()) { + return; + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); + sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); + } else { + sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); + sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java new file mode 100644 index 000000000..05a2a24e9 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java @@ -0,0 +1,99 @@ +package sqlancer.stonedb.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBInsertGenerator extends AbstractInsertGenerator { + private final StoneDBGlobalState globalState; + // which table to insert into + private final StoneDBTable table; + // which subset columns of the table to add values + private final List columns; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBInsertGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + table = globalState.getSchema().getRandomTable(); + columns = table.getRandomNonEmptyColumnSubset(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBInsertGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("INSERT"); + if (Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("LOW_PRIORITY", "DELAYED", "HIGH_PRIORITY")); + } + if (Randomly.getBoolean()) { + sb.append(" IGNORE"); + } + sb.append(" INTO "); + sb.append(table.getName()); + appendPartition(); + appendColumnsAndValues(columns); + return new SQLQueryAdapter(sb.toString(), errors); + } + + private void appendPartition() { + + } + + private void appendColumnsAndValues(List columns) { + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(Randomly.fromOptions(" VALUES ", " VALUE ")); + appendValues(); + } + + private void appendValues() { + int nrRows; + if (Randomly.getBoolean()) { + nrRows = 1; + } else { + nrRows = 1 + Randomly.smallNumber(); + } + for (int row = 0; row < nrRows; row++) { + appendOneValue(row); + } + } + + private void appendOneValue(int nrRow) { + if (nrRow != 0) { + sb.append(", "); + } + sb.append("("); + for (int c = 0; c < columns.size(); c++) { + if (c != 0) { + sb.append(", "); + } + sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState).generateConstant())); + + } + sb.append(")"); + } + + @Override + protected void insertValue(StoneDBColumn column) { + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append("DEFAULT"); + } else { + sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java new file mode 100644 index 000000000..55ebd0715 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -0,0 +1,94 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; + +public class StoneDBTableAlterGenerator { + private final StoneDBGlobalState globalState; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN, CHANGE_COLUMN, RENAME_COLUMN + } + + public StoneDBTableAlterGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBTableAlterGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("ALTER TABLE "); + StoneDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) + .setColumns(table.getColumns()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(StoneDBCompositeDataType.getRandomWithoutNull()); + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" FIRST"); + } else { + sb.append(" AFTER "); + sb.append(table.getRandomColumn().getName()); + } + } + break; + case DROP_COLUMN: + sb.append(Randomly.fromOptions("DROP COLUMN ", "DROP ")); + sb.append(table.getRandomColumn().getName()); + break; + case ALTER_COLUMN: + sb.append(Randomly.fromOptions("ALTER COLUMN ", "ALTER ")); + sb.append(table.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(" SET DEFAULT ").append(generator.generateExpression()); + } else { + sb.append(" DROP DEFAULT"); + } + if (Randomly.getBoolean()) { + sb.append(" SET ").append(Randomly.fromOptions("VISIBLE", "INVISIBLE")); + } + break; + case CHANGE_COLUMN: + sb.append(Randomly.fromOptions("CHANGE COLUMN ", "CHANGE ")); + String oldColumnName = table.getRandomColumn().getName(); + String newColumnName = table.getFreeColumnName(); + sb.append(oldColumnName).append(" ").append(newColumnName); + sb.append(" "); + sb.append(StoneDBCompositeDataType.getRandomWithoutNull()); + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + sb.append(" FIRST"); + } else { + sb.append(" AFTER "); + sb.append(table.getRandomColumn().getName()); + } + } + break; + case RENAME_COLUMN: + sb.append("RENAME COLUMN"); + sb.append(table.getRandomColumn().getName()); + sb.append(" TO "); + sb.append(table.getFreeColumnName()); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java new file mode 100644 index 000000000..134e0edfe --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java @@ -0,0 +1,357 @@ +package sqlancer.stonedb.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; + +public class StoneDBTableCreateGenerator { + // the name of the table to create + private final String tableName; + private final StoneDBSchema schema; + // the name of the columns in the table + private final List columns = new ArrayList<>(); + private final boolean allowPrimaryKey; + private boolean setPrimaryKey; + private final StringBuilder sb = new StringBuilder(); + private final Randomly r; + + public StoneDBTableCreateGenerator(StoneDBGlobalState globalState, String tableName) { + this.tableName = tableName; + this.schema = globalState.getSchema(); + allowPrimaryKey = Randomly.getBoolean(); + this.r = globalState.getRandomly(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String tableName) { + return new StoneDBTableCreateGenerator(globalState, tableName).getQuery(); + } + + public SQLQueryAdapter getQuery() { + ExpectedErrors errors = new ExpectedErrors(); + sb.append(Randomly.fromOptions("CREATE TABLE", "CREATE TEMPORARY TABLE")); + if (Randomly.getBoolean()) { + sb.append(" IF NOT EXISTS "); + } + sb.append(tableName); + // ues link statement + if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { + sb.append(" LIKE "); + sb.append(schema.getRandomTable().getName()); + return new SQLQueryAdapter(sb.toString(), true); + } else { + appendColumns(); + sb.append(" "); + appendTableOptions(); + addCommonErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + } + + private enum TableOptions { + AUTO_INCREMENT, AVG_ROW_LENGTH, CHECKSUM, COMPRESSION, DELAY_KEY_WRITE, /* ENCRYPTION, */ INSERT_METHOD, + KEY_BLOCK_SIZE, MAX_ROWS, MIN_ROWS, PACK_KEYS, STATS_AUTO_RECALC, STATS_PERSISTENT, STATS_SAMPLE_PAGES; + + public static List getRandomTableOptions() { + List options; + // try to ensure that usually, only a few of these options are generated + if (Randomly.getBooleanWithSmallProbability()) { + options = Randomly.subset(TableOptions.values()); + } else { + if (Randomly.getBoolean()) { + options = Collections.emptyList(); + } else { + options = Randomly.nonEmptySubset(Arrays.asList(TableOptions.values()), Randomly.smallNumber()); + } + } + return options; + } + } + + private void appendTableOptions() { + List tableOptions = TableOptions.getRandomTableOptions(); + int i = 0; + for (TableOptions o : tableOptions) { + if (i++ != 0) { + sb.append(", "); + } + switch (o) { + case AUTO_INCREMENT: + sb.append("AUTO_INCREMENT = "); + sb.append(r.getPositiveInteger()); + break; + // The valid range for avg_row_length is [0,4294967295] + case AVG_ROW_LENGTH: + sb.append("AVG_ROW_LENGTH = "); + sb.append(r.getLong(0, 4294967295L + 1)); + break; + case CHECKSUM: + sb.append("CHECKSUM = 1"); + break; + case COMPRESSION: + sb.append("COMPRESSION = '"); + sb.append(Randomly.fromOptions("ZLIB", "LZ4", "NONE")); + sb.append("'"); + break; + case DELAY_KEY_WRITE: + sb.append("DELAY_KEY_WRITE = "); + sb.append(Randomly.fromOptions(0, 1)); + break; + case INSERT_METHOD: + sb.append("INSERT_METHOD = "); + sb.append(Randomly.fromOptions("NO", "FIRST", "LAST")); + break; + // The valid range for key_block_size is [0,65535] + case KEY_BLOCK_SIZE: + sb.append("KEY_BLOCK_SIZE = "); + sb.append(r.getInteger(0, 65535 + 1)); + break; + case MAX_ROWS: + sb.append("MAX_ROWS = "); + sb.append(r.getLong(0, Long.MAX_VALUE)); + break; + case MIN_ROWS: + sb.append("MIN_ROWS = "); + sb.append(r.getLong(1, Long.MAX_VALUE)); + break; + case PACK_KEYS: + sb.append("PACK_KEYS = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); + break; + case STATS_AUTO_RECALC: + sb.append("STATS_AUTO_RECALC = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); + break; + case STATS_PERSISTENT: + sb.append("STATS_PERSISTENT = "); + sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); + break; + case STATS_SAMPLE_PAGES: + sb.append("STATS_SAMPLE_PAGES = "); + sb.append(r.getInteger(1, Short.MAX_VALUE)); + break; + default: + throw new AssertionError(o); + } + } + } + + private void addCommonErrors(ExpectedErrors list) { + list.add("The storage engine for the table doesn't support"); + list.add("doesn't have this option"); + list.add("must include all columns"); + list.add("not allowed type for this type of partitioning"); + list.add("doesn't support BLOB/TEXT columns"); + list.add("A BLOB field is not allowed in partition function"); + list.add("Too many keys specified; max 1 keys allowed"); + list.add("The total length of the partitioning fields is too large"); + list.add("Got error -1 - 'Unknown error -1' from storage engine"); + } + + private void appendColumns() { + sb.append("("); + for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(i); + } + sb.append(")"); + } + + private void appendColumn(int columnId) { + String columnName = DBMSCommon.createColumnName(columnId); + columns.add(columnName); + sb.append(columnName); + appendColumnDefinition(); + } + + private void appendColumnDefinition() { + sb.append(" "); + StoneDBDataType randomType = StoneDBDataType.getRandomWithoutNull(); + appendType(randomType); + sb.append(" "); + appendColumnOption(randomType); + } + + private enum ColumnOptions { + NULL_OR_NOT_NULL, UNIQUE, COMMENT, COLUMN_FORMAT, STORAGE, PRIMARY_KEY + } + + private void appendColumnOption(StoneDBDataType type) { + boolean isTextType = type == StoneDBDataType.VARCHAR; + boolean isNull = false; + boolean columnHasPrimaryKey = false; + List columnOptions = Randomly.subset(ColumnOptions.values()); + // boolean tableHasNullableColumn; + // if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { + // tableHasNullableColumn = true; + // } + if (isTextType) { + // TODO: restriction due to the limited key length + columnOptions.remove(ColumnOptions.PRIMARY_KEY); + columnOptions.remove(ColumnOptions.UNIQUE); + } + for (ColumnOptions o : columnOptions) { + sb.append(" "); + switch (o) { + case NULL_OR_NOT_NULL: + // PRIMARY KEYs cannot be NULL + if (!columnHasPrimaryKey) { + if (Randomly.getBoolean()) { + sb.append("NULL"); + } + // tableHasNullableColumn = true; + isNull = true; + } else { + sb.append("NOT NULL"); + } + break; + case UNIQUE: + sb.append("UNIQUE"); + if (Randomly.getBoolean()) { + sb.append(" KEY"); + } + break; + case COMMENT: + // TODO: generate randomly + sb.append(String.format("COMMENT '%s' ", "asdf")); + break; + case COLUMN_FORMAT: + sb.append("COLUMN_FORMAT "); + sb.append(Randomly.fromOptions("FIXED", "DYNAMIC", "DEFAULT")); + break; + case STORAGE: + sb.append("STORAGE "); + sb.append(Randomly.fromOptions("DISK", "MEMORY")); + break; + case PRIMARY_KEY: + // PRIMARY KEYs cannot be NULL + if (allowPrimaryKey && !setPrimaryKey && !isNull) { + sb.append("PRIMARY KEY"); + setPrimaryKey = true; + columnHasPrimaryKey = true; + } + break; + default: + throw new AssertionError(); + } + } + } + + private void appendType(StoneDBDataType randomType) { + switch (randomType) { + case TINYINT: + sb.append("TINYINT"); + // sb.append(r.getInteger(-128, 127)); + break; + case SMALLINT: + sb.append("SMALLINT"); + // sb.append(r.getInteger(-32768, 32767)); + break; + case MEDIUMINT: + sb.append("MEDIUMINT"); + // sb.append(r.getInteger(-8388608, 8388607)); + break; + case INT: + sb.append("INT"); + // sb.append(r.getInteger(-2147483647, 2147483647)); + break; + case BIGINT: + sb.append("BIGINT"); + // sb.append(r.getBigInteger(new BigInteger("-9223372036854775806"), new + // BigInteger("9223372036854775807"))); + break; + case FLOAT: + sb.append("FLOAT"); + optionallyAddPrecisionAndScale(sb); + break; + case DOUBLE: + sb.append("DOUBLE"); + optionallyAddPrecisionAndScale(sb); + break; + case DECIMAL: + sb.append("DECIMAL"); // The default value is P(10,0); + break; + case YEAR: + sb.append("YEAR"); + break; + case TIME: + sb.append("TIME"); + break; + case DATE: + sb.append("DATE"); + break; + case DATETIME: + sb.append("DATETIME"); + break; + case TIMESTAMP: + sb.append("TIMESTAMP"); + break; + case CHAR: + sb.append("CHAR"); + break; + case VARCHAR: + sb.append("VARCHAR("); + sb.append(r.getInteger(0, 65535)); + sb.append(")"); + break; + case TINYTEXT: + sb.append("TINYTEXT"); + break; + case TEXT: + sb.append("TEXT"); + break; + case MEDIUMTEXT: + sb.append("MEDIUMTEXT"); + break; + case LONGTEXT: + sb.append("LONGTEXT"); + break; + case BINARY: + sb.append("BINARY"); + break; + case VARBINARY: + sb.append("VARBINARY"); + break; + case TINYBLOB: + sb.append("TINYBLOB"); + break; + case BLOB: + sb.append("BLOB"); + break; + case MEDIUMBLOB: + sb.append("MEDIUMBLOB"); + break; + case LONGBLOB: + sb.append("LONGBLOB"); + break; + default: + throw new AssertionError(); + } + } + + public static void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean()) { + sb.append("("); + // The maximum number of digits (M) for DECIMAL is 65 + long m = Randomly.getNotCachedInteger(1, 65); + sb.append(m); + sb.append(", "); + // The maximum number of supported decimals (D) is 30 + long nCandidate = Randomly.getNotCachedInteger(1, 30); + // For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'c0'). + long n = Math.min(nCandidate, m); + sb.append(n); + sb.append(")"); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java new file mode 100644 index 000000000..bff098a64 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java @@ -0,0 +1,69 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBTableUpdateGenerator { + private final StoneDBGlobalState globalState; + // which table to insert into + private final StoneDBTable table; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBTableUpdateGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + table = globalState.getSchema().getRandomTable(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBTableUpdateGenerator(globalState).getQuery(); + } + + public SQLQueryAdapter getQuery() { + sb.append("UPDATE"); + sb.append(Randomly.fromOptions(" ", " LOW_PRIORITY ", " LOW_PRIORITY IGNORE ", " IGNORE ")); + sb.append(table.getName()); + sb.append("SET "); + appendAssignmentList(); + if (Randomly.getBoolean()) { + appendWhereCondition(); + } + if (Randomly.getBoolean()) { + appendOrderBy(); + } + if (Randomly.getBoolean()) { + appendLimit(); + } + return new SQLQueryAdapter(sb.toString(), errors); + } + + public void appendAssignmentList() { + for (int i = 0; i < new Randomly().getInteger(1, table.getColumns().size()); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(table.getColumns().get(i)); + sb.append(" = "); + sb.append(StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType + .getRandomValue(table.getColumns().get(i).getType().getPrimitiveDataType()))); + } + } + + private void appendWhereCondition() { + } + + private void appendOrderBy() { + sb.append(" ORDER BY "); + sb.append(table.getRandomColumn().getName()); + } + + private void appendLimit() { + sb.append(" LIMIT "); + sb.append(new Randomly().getInteger(0, (int) table.getNrRows(globalState))); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java new file mode 100644 index 000000000..ac998b180 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -0,0 +1,16 @@ +package sqlancer.stonedb.oracle; + +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; + +public class StoneDBNoRECOracle extends NoRECBase implements TestOracle { + public StoneDBNoRECOracle(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java b/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java new file mode 100644 index 000000000..96deda89e --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java @@ -0,0 +1,16 @@ +package sqlancer.stonedb.oracle; + +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; + +public class StoneDBTLPOracle extends NoRECBase implements TestOracle { + public StoneDBTLPOracle(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + + } +} From 37c33981c7fdc3039fe7ab6db15e7ce6c577228f Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 26 Jun 2023 20:54:28 +0800 Subject: [PATCH 362/856] CockroachDB: optimize code for CERT (#839) * cockroachdb: fill all tables * cockroachdb: flat outer join * cockroachdb: add utility functions * cockroachdb: support inserting to a specific table * cockorachdb: optimize generators * randomly: assert --- src/sqlancer/Randomly.java | 6 ++ .../cockroachdb/CockroachDBErrors.java | 3 +- .../cockroachdb/CockroachDBProvider.java | 12 ++++ .../cockroachdb/CockroachDBSchema.java | 4 ++ .../CockroachDBToStringVisitor.java | 61 +++++++++++-------- .../cockroachdb/ast/CockroachDBJoin.java | 52 ++++++---------- .../gen/CockroachDBInsertGenerator.java | 6 +- ...CockroachDBSetClusterSettingGenerator.java | 3 +- .../gen/CockroachDBShowGenerator.java | 2 +- .../oracle/CockroachDBNoRECOracle.java | 21 +------ 10 files changed, 88 insertions(+), 82 deletions(-) diff --git a/src/sqlancer/Randomly.java b/src/sqlancer/Randomly.java index 2ceacfb30..8494c189a 100644 --- a/src/sqlancer/Randomly.java +++ b/src/sqlancer/Randomly.java @@ -135,6 +135,12 @@ public static List nonEmptySubset(List columns, int nr) { return extractNrRandomColumns(columns, nr); } + public static List nonEmptySubsetLeast(List columns, int min) { + int nr = getNextInt(min, columns.size() + 1); + assert nr <= columns.size(); + return extractNrRandomColumns(columns, nr); + } + public static List nonEmptySubsetPotentialDuplicates(List columns) { List arr = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index e282981dc..7cb738ee8 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -19,7 +19,8 @@ public static void addExpressionErrors(ExpectedErrors errors) { } errors.add("exceeds supported timestamp bounds"); - + errors.add("expected STORED COMPUTED COLUMN expression to have type bytes"); + errors.add("volatile functions are not allowed in STORED COMPUTED COLUMN"); errors.add("cannot cast negative integer to bit varying with unbounded width"); errors.add("negative value for LIMIT"); diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index c9e0d24a9..c2497e5c8 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -7,6 +7,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -340,4 +341,15 @@ protected void executeMutator(int index, CockroachDBGlobalState globalState) thr globalState.executeStatement(queryMutateTable); } + @Override + public boolean addRowsToAllTables(CockroachDBGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (CockroachDBTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = CockroachDBInsertGenerator.insert(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index c245f4cce..708afef92 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -208,6 +208,10 @@ public CockroachDBTables getRandomTableNonEmptyTables() { return new CockroachDBTables(Randomly.nonEmptySubset(getDatabaseTables())); } + public CockroachDBTables getRandomTableNonEmptyTables(int nr) { + return new CockroachDBTables(Randomly.nonEmptySubsetLeast(getDatabaseTables(), nr)); + } + private static CockroachDBCompositeDataType getColumnType(String typeString) { if (typeString.endsWith("[]")) { String substring = typeString.substring(0, typeString.length() - 2); diff --git a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java index 17a09a401..1c6fe0f36 100644 --- a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java @@ -142,54 +142,65 @@ public void visit(CockroachDBJoin join) { switch (join.getJoinType()) { case INNER: sb.append(" INNER "); - potentiallyAddHint(); + potentiallyAddHint(false); sb.append("JOIN "); visit(join.getRightTable()); sb.append(" ON "); visit(join.getOnCondition()); break; - case NATURAL: - sb.append(" NATURAL "); - // potentiallyAddHint(); + case LEFT: + sb.append(" LEFT"); + sb.append(" OUTER "); + potentiallyAddHint(true); sb.append("JOIN "); visit(join.getRightTable()); + sb.append(" ON "); + visit(join.getOnCondition()); break; - case CROSS: - sb.append(" CROSS "); - potentiallyAddHint(); + case RIGHT: + sb.append(" RIGHT"); + sb.append(" OUTER "); + potentiallyAddHint(true); sb.append("JOIN "); visit(join.getRightTable()); + sb.append(" ON "); + visit(join.getOnCondition()); break; - case OUTER: - sb.append(" "); - switch (join.getOuterType()) { - case FULL: - sb.append("FULL"); - break; - case LEFT: - sb.append("LEFT"); - break; - case RIGHT: - sb.append("RIGHT"); - break; - default: - throw new AssertionError(); - } + case FULL: + sb.append(" FULL"); sb.append(" OUTER "); - potentiallyAddHint(); + potentiallyAddHint(true); sb.append("JOIN "); visit(join.getRightTable()); sb.append(" ON "); visit(join.getOnCondition()); break; + case CROSS: + sb.append(" CROSS "); + potentiallyAddHint(false); + sb.append("JOIN "); + visit(join.getRightTable()); + break; + case NATURAL: + sb.append(" NATURAL "); + // potentiallyAddHint(false); + sb.append("JOIN "); + visit(join.getRightTable()); + break; default: throw new AssertionError(); } } - private void potentiallyAddHint() { + private void potentiallyAddHint(boolean isOuter) { if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions("HASH", "MERGE", "LOOKUP")); + return; + } else { + if (isOuter) { + sb.append(Randomly.fromOptions("HASH", "MERGE", "LOOKUP")); + } else { + sb.append(Randomly.fromOptions("HASH", "MERGE")); + } sb.append(" "); } } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java index 740a7901c..7259eed22 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java @@ -6,26 +6,17 @@ public class CockroachDBJoin implements CockroachDBExpression { private final CockroachDBExpression leftTable; private final CockroachDBExpression rightTable; - private final JoinType joinType; - private final CockroachDBExpression onCondition; - private OuterType outerType; + private JoinType joinType; + private CockroachDBExpression onCondition; public enum JoinType { - INNER, NATURAL, CROSS, OUTER; + INNER, LEFT, RIGHT, FULL, CROSS, NATURAL; public static JoinType getRandom() { return Randomly.fromOptions(values()); } } - public enum OuterType { - FULL, LEFT, RIGHT; - - public static OuterType getRandom() { - return Randomly.fromOptions(values()); - } - } - public CockroachDBJoin(CockroachDBExpression leftTable, CockroachDBExpression rightTable, JoinType joinType, CockroachDBExpression whereCondition) { this.leftTable = leftTable; @@ -42,35 +33,28 @@ public CockroachDBExpression getRightTable() { return rightTable; } - public JoinType getJoinType() { - return joinType; - } - - public CockroachDBExpression getOnCondition() { - return onCondition; - } - - public static CockroachDBJoin createNaturalJoin(CockroachDBExpression left, CockroachDBExpression right) { - return new CockroachDBJoin(left, right, JoinType.NATURAL, null); + public void setJoinType(JoinType joinType) { + this.joinType = joinType; } - public static CockroachDBJoin createCrossJoin(CockroachDBExpression left, CockroachDBExpression right) { - return new CockroachDBJoin(left, right, JoinType.CROSS, null); + public JoinType getJoinType() { + return joinType; } - public static CockroachDBJoin createOuterJoin(CockroachDBExpression left, CockroachDBExpression right, - OuterType type, CockroachDBExpression onClause) { - CockroachDBJoin join = new CockroachDBJoin(left, right, JoinType.OUTER, onClause); - join.setOuterType(type); - return join; + public void setOnCondition(CockroachDBExpression onCondition) { + this.onCondition = onCondition; } - private void setOuterType(OuterType outerType) { - this.outerType = outerType; + public CockroachDBExpression getOnCondition() { + return onCondition; } - public OuterType getOuterType() { - return outerType; + public static CockroachDBJoin createJoin(CockroachDBExpression left, CockroachDBExpression right, JoinType type, + CockroachDBExpression onClause) { + if (type.compareTo(JoinType.CROSS) >= 0) { + return new CockroachDBJoin(left, right, type, null); + } else { + return new CockroachDBJoin(left, right, type, onClause); + } } - } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java index 47de622d8..decbe927f 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java @@ -18,6 +18,11 @@ private CockroachDBInsertGenerator() { } public static SQLQueryAdapter insert(CockroachDBGlobalState globalState) { + CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + return insert(globalState, table); + } + + public static SQLQueryAdapter insert(CockroachDBGlobalState globalState, CockroachDBTable table) { ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addExpressionErrors(errors); // e.g., caused by computed columns @@ -32,7 +37,6 @@ public static SQLQueryAdapter insert(CockroachDBGlobalState globalState) { errors.add("foreign key violation"); errors.add("multi-part foreign key"); StringBuilder sb = new StringBuilder(); - CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean isUpsert = Randomly.getBoolean(); if (!isUpsert) { sb.append("INSERT INTO "); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index 42740506d..cfc89562a 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -20,7 +20,8 @@ private enum CockroachDBClusterSetting { RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), SQL_STATS_HISTOGRAM_COLLECTION_ENABLED("sql.stats.histogram_collection.enabled", - CockroachDBSetSessionGenerator::onOff); + CockroachDBSetSessionGenerator::onOff), + HISTOGRAM_COLLECT("sql.stats.histogram_collection.enabled", CockroachDBSetSessionGenerator::onOff); private Function f; private String name; diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java index 09abbfbd0..9b84d9c94 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBShowGenerator.java @@ -23,7 +23,7 @@ public static SQLQueryAdapter show(CockroachDBGlobalState globalState) { case EXPERIMENTAL_FINGERPRINTS: sb.append("SHOW EXPERIMENTAL_FINGERPRINTS FROM TABLE "); sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); - errors.add("as type bytes: bytea encoded value ends with incomplete escape sequence"); + errors.add("bytea encoded value ends with incomplete escape sequence"); errors.add("invalid bytea escape sequence"); break; case DATABASES: diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 6ab133b1c..c627b9151 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -19,7 +19,6 @@ import sqlancer.cockroachdb.ast.CockroachDBColumnReference; import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBJoin.OuterType; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; @@ -75,24 +74,8 @@ public static List getJoins(List t columns.addAll(rightTable.getTable().getColumns()); CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) .setColumns(columns); - switch (CockroachDBJoin.JoinType.getRandom()) { - case INNER: - joinExpressions.add(new CockroachDBJoin(leftTable, rightTable, CockroachDBJoin.JoinType.INNER, - joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); - break; - case NATURAL: - joinExpressions.add(CockroachDBJoin.createNaturalJoin(leftTable, rightTable)); - break; - case CROSS: - joinExpressions.add(CockroachDBJoin.createCrossJoin(leftTable, rightTable)); - break; - case OUTER: - joinExpressions.add(CockroachDBJoin.createOuterJoin(leftTable, rightTable, OuterType.getRandom(), - joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); - break; - default: - throw new AssertionError(); - } + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); } return joinExpressions; } From 5a69cad46238a075cb91715938defb55d138ec30 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 26 Jun 2023 20:55:39 +0800 Subject: [PATCH 363/856] Refactor: rename column variable names in InsertGenerators (#838) --- src/sqlancer/common/gen/AbstractInsertGenerator.java | 2 +- src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java | 2 +- src/sqlancer/h2/H2InsertGenerator.java | 2 +- src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/common/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java index 61357fd10..1a0b2a997 100644 --- a/src/sqlancer/common/gen/AbstractInsertGenerator.java +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -24,6 +24,6 @@ protected void insertColumns(List columns) { } } - protected abstract void insertValue(C tiDBColumn); + protected abstract void insertValue(C column); } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 1e5eb60b9..4fd3f390c 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -41,7 +41,7 @@ private SQLQueryAdapter generate() { } @Override - protected void insertValue(DuckDBColumn tiDBColumn) { + protected void insertValue(DuckDBColumn columnDuckDB) { // TODO: select a more meaningful value if (Randomly.getBooleanWithRatherLowProbability()) { sb.append("DEFAULT"); diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index cfb17846a..c4e559b0e 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -56,7 +56,7 @@ private SQLQueryAdapter generate() { } @Override - protected void insertValue(H2Column tiDBColumn) { + protected void insertValue(H2Column columnH2) { sb.append(H2ToStringVisitor.asString(gen.generateConstant())); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java index a27ec9c92..a1159d310 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -57,7 +57,7 @@ protected void insertColumns(List columns) { } @Override - protected void insertValue(YCQLColumn tiDBColumn) { + protected void insertValue(YCQLColumn columnYCQL) { // TODO: select a more meaningful value sb.append(YCQLToStringVisitor.asString(new YCQLExpressionGenerator(globalState).generateConstant())); } From 03ddf95b1ec9ee016c4298d913a0da2f2d236b1c Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 26 Jun 2023 23:32:07 +0800 Subject: [PATCH 364/856] cockroachdb: cert init (#842) * cockroachdb: cert init * tidb: getRandomExcept Join --- .../cockroachdb/CockroachDBOptions.java | 12 + .../cockroachdb/CockroachDBProvider.java | 19 +- .../cockroachdb/ast/CockroachDBJoin.java | 8 + .../oracle/CockroachDBCERTOracle.java | 275 ++++++++++++++++++ src/sqlancer/tidb/ast/TiDBJoin.java | 7 + src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 10 +- 6 files changed, 320 insertions(+), 11 deletions(-) create mode 100644 src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index c41b018ee..e50a30c51 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -12,6 +12,7 @@ import sqlancer.OracleFactory; import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; @@ -88,6 +89,17 @@ public TestOracle create(CockroachDBGlobalState globalSt oracles.add(new CockroachDBTLPDistinctOracle(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + CERT { + @Override + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } }; } diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index c2497e5c8..7ecd4de9c 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -36,6 +36,7 @@ import sqlancer.cockroachdb.gen.CockroachDBTruncateGenerator; import sqlancer.cockroachdb.gen.CockroachDBUpdateGenerator; import sqlancer.cockroachdb.gen.CockroachDBViewGenerator; +import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; @@ -130,13 +131,13 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio QueryManager manager = globalState.getManager(); MainOptions options = globalState.getOptions(); List standardSettings = new ArrayList<>(); - standardSettings.add("--Don't send automatic bug reports\n" - + "SET CLUSTER SETTING debug.panic_on_failed_assertions = true;"); + standardSettings.add("--Don't send automatic bug reports"); + standardSettings.add("SET CLUSTER SETTING debug.panic_on_failed_assertions = true;"); standardSettings.add("SET CLUSTER SETTING diagnostics.reporting.enabled = false;"); standardSettings.add("SET CLUSTER SETTING diagnostics.reporting.send_crash_reports = false;"); - standardSettings.add("-- Disable the collection of metrics and hope that it helps performance\n" - + "SET CLUSTER SETTING sql.metrics.statement_details.enabled = 'off'"); + standardSettings.add("-- Disable the collection of metrics and hope that it helps performance"); + standardSettings.add("SET CLUSTER SETTING sql.metrics.statement_details.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING sql.metrics.statement_details.plan_collection.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING sql.stats.automatic_collection.enabled = 'off'"); standardSettings.add("SET CLUSTER SETTING timeseries.storage.enabled = 'off'"); @@ -250,6 +251,16 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio } total--; } + + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 && globalState + .getDbmsSpecificOptions().getTestOracleFactory().get(0).create(globalState) instanceof CockroachDBCERTOracle) { + // Enfore statistic collected for all tables + ExpectedErrors errors = new ExpectedErrors(); + CockroachDBErrors.addExpressionErrors(errors); + for (CockroachDBTable table : globalState.getSchema().getDatabaseTables()) { + globalState.executeStatement(new SQLQueryAdapter("ANALYZE " + table.getName() + ";", errors)); + } + } } @Override diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java index 7259eed22..b53c4f681 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java @@ -1,5 +1,7 @@ package sqlancer.cockroachdb.ast; +import java.util.Arrays; + import sqlancer.Randomly; public class CockroachDBJoin implements CockroachDBExpression { @@ -15,6 +17,12 @@ public enum JoinType { public static JoinType getRandom() { return Randomly.fromOptions(values()); } + + public static JoinType getRandomExcept(JoinType... exclude) { + JoinType[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(JoinType[]::new); + return Randomly.fromOptions(values); + } } public CockroachDBJoin(CockroachDBExpression leftTable, CockroachDBExpression rightTable, JoinType joinType, diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java new file mode 100644 index 000000000..010188431 --- /dev/null +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -0,0 +1,275 @@ +package sqlancer.cockroachdb.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.cockroachdb.CockroachDBCommon; +import sqlancer.cockroachdb.CockroachDBErrors; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; +import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation; +import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation.CockroachDBBinaryLogicalOperator; +import sqlancer.cockroachdb.ast.CockroachDBColumnReference; +import sqlancer.cockroachdb.ast.CockroachDBExpression; +import sqlancer.cockroachdb.ast.CockroachDBJoin; +import sqlancer.cockroachdb.ast.CockroachDBJoin.JoinType; +import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.ast.CockroachDBTableReference; +import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CERTOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; + +public class CockroachDBCERTOracle extends CERTOracleBase + implements TestOracle { + private CockroachDBExpressionGenerator gen; + private CockroachDBSelect select; + + public CockroachDBCERTOracle(CockroachDBGlobalState globalState) { + super(globalState); + CockroachDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + queryPlan1Sequences = new ArrayList<>(); + queryPlan2Sequences = new ArrayList<>(); + + // Randomly generate a query + CockroachDBTables tables = state.getSchema().getRandomTableNonEmptyTables(2); + List tableList = CockroachDBCommon.getTableReferences( + tables.getTables().stream().map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList())); + gen = new CockroachDBExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new CockroachDBColumnReference(c)).collect(Collectors.toList())); + select = new CockroachDBSelect(); + select.setFetchColumns(fetchColumns); + select.setFromList(tableList); + select.setDistinct(Randomly.getBoolean()); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } + } + + // Set the join. + List joinExpressions = getJoins(tableList, state); + select.setJoinList(joinExpressions); + + // Get the result of the first query + String queryString1 = CockroachDBVisitor.asString(select); + int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + // Mutate the query + // Disable limit due to its false positive + boolean increase = mutate(); + + // Get the result of the second query + String queryString2 = CockroachDBVisitor.asString(select); + int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + // Check structural equivalence + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check the results + if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { + throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 + + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + } + } + + private List getJoins(List tableList, + CockroachDBGlobalState globalState) throws AssertionError { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getPercentage() < 0.8) { + CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); + CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, + CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + return joinExpressions; + } + + @Override + protected boolean mutateJoin() { + if (select.getJoinList().isEmpty()) { + return false; + } + + CockroachDBJoin join = (CockroachDBJoin) Randomly.fromList(select.getJoinList()); + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((CockroachDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((CockroachDBTableReference) join.getRightTable()).getTable().getColumns()); + CockroachDBExpressionGenerator joinGen2 = new CockroachDBExpressionGenerator(state).setColumns(columns); + join.setOnCondition(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); + } + + JoinType newJoinType = CockroachDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation + // between LEFT and RIGHT + // join + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); + } else { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); + } + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + return increase; + } + + @Override + protected boolean mutateDistinct() { + boolean increase = select.isDistinct(); + select.setDistinct(!select.isDistinct()); + return increase; + } + + @Override + protected boolean mutateWhere() { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } + return increase; + } + + @Override + protected boolean mutateGroupBy() { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + @Override + protected boolean mutateHaving() { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + @Override + protected boolean mutateAnd() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + @Override + protected boolean mutateOr() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + @Override + protected boolean mutateLimit() { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(gen.generateConstant(CockroachDBDataType.INT.get())); + } + return increase; + } + + private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + throws AssertionError, SQLException { + int row = -1; + String explainQuery = "EXPLAIN (VERBOSE) " + selectStr; + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String content = rs.getString(1); + if (content.contains("count:")) { + try { + int number = Integer.parseInt(content.split("count: ")[1].split(" ")[0].replace(",", "")); + if (row == -1) { + row = number; + } + } catch (Exception e) { // To avoid the situation that no number is found + } + } + if (content.contains("• ")) { + String operation = content.split("• ")[1].split(" ")[0]; + queryPlanSequences.add(operation); + } + } + } + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + if (row == -1) { + throw new IgnoreMeException(); + } + return row; + } + +} diff --git a/src/sqlancer/tidb/ast/TiDBJoin.java b/src/sqlancer/tidb/ast/TiDBJoin.java index e48c7a6f7..f4496a736 100644 --- a/src/sqlancer/tidb/ast/TiDBJoin.java +++ b/src/sqlancer/tidb/ast/TiDBJoin.java @@ -1,6 +1,7 @@ package sqlancer.tidb.ast; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import sqlancer.Randomly; @@ -22,6 +23,12 @@ public enum JoinType { public static JoinType getRandom() { return Randomly.fromOptions(values()); } + + public static JoinType getRandomExcept(JoinType... exclude) { + JoinType[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(JoinType[]::new); + return Randomly.fromOptions(values); + } } public enum NaturalJoinType { diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java index 1bef4ac96..4e56dd5fa 100644 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -129,14 +128,11 @@ protected boolean mutateJoin() { if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation // between LEFT and RIGHT // join - newJoinType = Randomly.fromOptions(Arrays.stream(JoinType.values()) - .filter(j -> j != JoinType.LEFT && j != JoinType.RIGHT && j != JoinType.NATURAL) - .toArray(JoinType[]::new)); + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); } else { - newJoinType = Randomly.fromOptions(Arrays.stream(JoinType.values()) - .filter(j -> j != join.getJoinType() && j != JoinType.NATURAL).toArray(JoinType[]::new)); + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); } - + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); join.setJoinType(newJoinType); if (newJoinType == JoinType.CROSS) { From 1aeb0f6447af76d72bfada610c2cb334f847ab42 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Wed, 28 Jun 2023 15:36:15 +0800 Subject: [PATCH 365/856] doc: add CERT (#843) * cockroachdb: formatting * doc: add CERT * cert: collect every tables for any cert oracle --- README.md | 6 ++++-- src/sqlancer/cockroachdb/CockroachDBProvider.java | 6 +++--- src/sqlancer/mysql/MySQLProvider.java | 7 +++---- src/sqlancer/tidb/TiDBProvider.java | 6 +++--- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index f22b00f4d..5c18c887d 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Man SQLancer operates in the following two phases: -1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. **New: we support query plan guided generation now. See Generation Approaches below.** -2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. +1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. **News: we support Query Plan Guidance (QPG) now. See Generation Approaches below.** +2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. **News: we support Cardinality Estimation Restriction Testing (CERT) oracle now. See Testing Approaches below.** # Getting Started @@ -40,6 +40,8 @@ If you launch SQLancer without parameters, available options and commands are di | Pivoted Query Synthesis (PQS) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described [here](https://arxiv.org/abs/2001.04174). PQS is the most powerful technique, but also requires more implementation effort than the other two techniques. It is currently unmaintained. | | Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | | Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | +| Cardinality Estimation Restriction Testing (CERT) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. It is described [here](https://arxiv.org/abs/2306.00355). It derives a query to a more restrict query, whose estimated cardinality should be no more than that for the original query. An violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. | + # Generation Approaches | Approach | Description | |----------|-------------| diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 7ecd4de9c..494c63a5b 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -19,6 +19,7 @@ import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; +import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.gen.CockroachDBCommentOnGenerator; @@ -36,7 +37,6 @@ import sqlancer.cockroachdb.gen.CockroachDBTruncateGenerator; import sqlancer.cockroachdb.gen.CockroachDBUpdateGenerator; import sqlancer.cockroachdb.gen.CockroachDBViewGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; @@ -252,8 +252,8 @@ public void generateDatabase(CockroachDBGlobalState globalState) throws Exceptio total--; } - if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 && globalState - .getDbmsSpecificOptions().getTestOracleFactory().get(0).create(globalState) instanceof CockroachDBCERTOracle) { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == CockroachDBOracleFactory.CERT)) { // Enfore statistic collected for all tables ExpectedErrors errors = new ExpectedErrors(); CockroachDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 0c3a1f2e7..201e0c799 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -21,6 +21,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; +import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; @@ -38,7 +39,6 @@ import sqlancer.mysql.gen.tblmaintenance.MySQLChecksum; import sqlancer.mysql.gen.tblmaintenance.MySQLOptimize; import sqlancer.mysql.gen.tblmaintenance.MySQLRepair; -import sqlancer.mysql.oracle.MySQLCERTOracle; @AutoService(DatabaseProvider.class) public class MySQLProvider extends SQLProviderAdapter { @@ -157,9 +157,8 @@ public void generateDatabase(MySQLGlobalState globalState) throws Exception { }); se.executeStatements(); - if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 - && globalState.getDbmsSpecificOptions().getTestOracleFactory().get(0) - .create(globalState) instanceof MySQLCERTOracle) { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == MySQLOracleFactory.CERT)) { // Enfore statistic collected for all tables ExpectedErrors errors = new ExpectedErrors(); MySQLErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 3701a4006..3e4133761 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -23,6 +23,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.gen.TiDBAlterTableGenerator; @@ -36,7 +37,6 @@ import sqlancer.tidb.gen.TiDBTableGenerator; import sqlancer.tidb.gen.TiDBUpdateGenerator; import sqlancer.tidb.gen.TiDBViewGenerator; -import sqlancer.tidb.oracle.TiDBCERTOracle; @AutoService(DatabaseProvider.class) public class TiDBProvider extends SQLProviderAdapter { @@ -139,8 +139,8 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { } } - if (globalState.getDbmsSpecificOptions().getTestOracleFactory().size() == 1 && globalState - .getDbmsSpecificOptions().getTestOracleFactory().get(0).create(globalState) instanceof TiDBCERTOracle) { + if (globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch((o) -> o == TiDBOracleFactory.CERT)) { // Disable strict Group By constraints for ROW oracle globalState.executeStatement(new SQLQueryAdapter( "SET @@sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION';")); From a90884f8199c0b7e0e9284ed9f07aeeec0d21084 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:57:35 +0800 Subject: [PATCH 366/856] Feat: add support for StoneDB NoREC oracle (#845) * feat: add StoneDBNoRECOracle support * refactor: change \n to System.lineSeparator() * refactor: improve code quality --- src/sqlancer/stonedb/StoneDBSchema.java | 11 ++ src/sqlancer/stonedb/ast/StoneDBJoin.java | 126 ++++++++++++++++++ src/sqlancer/stonedb/ast/StoneDBSelect.java | 17 +++ .../gen/StoneDBExpressionGenerator.java | 15 +++ .../stonedb/oracle/StoneDBNoRECOracle.java | 110 ++++++++++++++- 5 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/stonedb/ast/StoneDBJoin.java create mode 100644 src/sqlancer/stonedb/ast/StoneDBSelect.java diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java index eccf05c27..58170ce73 100644 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -12,6 +12,7 @@ import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -99,6 +100,12 @@ public boolean hasPrimaryKey() { } + public static class StoneDBTables extends AbstractTables { + public StoneDBTables(List tables) { + super(tables); + } + } + public static final class StoneDBIndex extends TableIndex { private StoneDBIndex(String indexName) { super(indexName); @@ -339,4 +346,8 @@ public static StoneDBCompositeDataType getRandomWithoutNull() { return new StoneDBCompositeDataType(type); } } + + public StoneDBTables getRandomTableNonEmptyTables() { + return new StoneDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } } diff --git a/src/sqlancer/stonedb/ast/StoneDBJoin.java b/src/sqlancer/stonedb/ast/StoneDBJoin.java new file mode 100644 index 000000000..d32d49f5b --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBJoin.java @@ -0,0 +1,126 @@ +package sqlancer.stonedb.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator; + +public class StoneDBJoin implements Node { + + public enum JoinType { + INNER, NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum NaturalJoinType { + FULL, LEFT, RIGHT; + + public static NaturalJoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + private NaturalJoinType naturalJoinType; + + public StoneDBJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node onCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = onCondition; + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + public NaturalJoinType getNaturalJoinType() { + return naturalJoinType; + } + + public void setNaturalJoinType(NaturalJoinType naturalJoinType) { + this.naturalJoinType = naturalJoinType; + } + + public static List> getJoins( + List> tableList, StoneDBGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + // get two tables to join + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + // store all columns in the above two tables + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + // create a join generator + StoneDBExpressionGenerator joinGen = new StoneDBExpressionGenerator(globalState).setColumns(columns); + switch (StoneDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(StoneDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + joinExpressions.add(StoneDBJoin.createNaturalJoin(leftTable, rightTable, NaturalJoinType.getRandom())); + break; + case LEFT: + joinExpressions + .add(StoneDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions + .add(StoneDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static StoneDBJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node onClause) { + return new StoneDBJoin(left, right, JoinType.RIGHT, onClause); + } + + public static StoneDBJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node onClause) { + return new StoneDBJoin(left, right, JoinType.LEFT, onClause); + } + + public static StoneDBJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node onClause) { + return new StoneDBJoin(left, right, JoinType.INNER, onClause); + } + + public static Node createNaturalJoin(TableReferenceNode left, + TableReferenceNode right, NaturalJoinType naturalJoinType) { + StoneDBJoin join = new StoneDBJoin(left, right, JoinType.NATURAL, null); + join.setNaturalJoinType(naturalJoinType); + return join; + } +} diff --git a/src/sqlancer/stonedb/ast/StoneDBSelect.java b/src/sqlancer/stonedb/ast/StoneDBSelect.java new file mode 100644 index 000000000..081106260 --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBSelect.java @@ -0,0 +1,17 @@ +package sqlancer.stonedb.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class StoneDBSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index cf578ecb0..59fb7e052 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -18,6 +18,7 @@ import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -35,6 +36,20 @@ private enum Expression { CASE } + public static class StoneDBCastOperation extends NewUnaryPostfixOperatorNode { + + public StoneDBCastOperation(Node expr, StoneDBCompositeDataType type) { + super(expr, new Operator() { + + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + } + + } + @Override public Node negatePredicate(Node predicate) { return null; diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index ac998b180..74ed75a91 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -1,16 +1,122 @@ package sqlancer.stonedb.oracle; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBSchema.StoneDBTables; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; public class StoneDBNoRECOracle extends NoRECBase implements TestOracle { - public StoneDBNoRECOracle(StoneDBGlobalState state) { - super(state); + + private final StoneDBSchema schema; + + public StoneDBNoRECOracle(StoneDBGlobalState globalState) { + super(globalState); + this.schema = globalState.getSchema(); } @Override public void check() throws Exception { + StoneDBTables randomTables = schema.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(state).setColumns(columns); + Node randomWhereCondition = gen.generateExpression(); + List tables = randomTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = StoneDBJoin.getJoins(tableList, state); + // get and check count + int secondCount = getUnoptimizedQueryCount(new ArrayList<>(tableList), randomWhereCondition, joins); + int firstCount = getOptimizedQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError(optimizedQueryString + "; -- " + firstCount + System.lineSeparator() + + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getUnoptimizedQueryCount(List> tableList, + Node randomWhereCondition, List> joins) throws SQLException { + StoneDBSelect select = new StoneDBSelect(); + Node asText = new NewPostfixTextNode<>(new StoneDBCastOperation( + new NewPostfixTextNode<>(randomWhereCondition, + " IS NOT NULL AND " + StoneDBToStringVisitor.asString(randomWhereCondition)), + new StoneDBCompositeDataType(StoneDBDataType.INT, 8)), "as count"); + select.setFetchColumns(List.of(asText)); + select.setFromList(tableList); + select.setJoinList(joins); + unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res"; + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError("error occurred when executing: \"" + unoptimizedQueryString + "\"", e); + } + if (rs == null) { + return -1; + } + int secondCount = 0; + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + private int getOptimizedQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, + List> joins) { + StoneDBSelect select = new StoneDBSelect(); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(new StoneDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + select.setJoinList(joins); + int firstCount; + try (Statement stat = con.createStatement()) { + optimizedQueryString = StoneDBToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + firstCount = rs.getFetchSize(); + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; } } From bff57106f3e103124f0fc010933c0235c46bd277 Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Tue, 4 Jul 2023 18:51:08 +0800 Subject: [PATCH 367/856] feat: Implement an AST based reducer prototype (#847) * feat: Implement an AST based reducer prototype + The reduer can run recursively visiting the ast-tree node and transforming the structure of the query + Provide some test cases at `test/reducer/TestASTBasedReducer.java` demonstrating the effect of the reducer. + Currently the reducer focus on only Select statements. Could be extended later. * fix format issues * feat: Add some support and do some modification according to the code review + Add support for String value shortening. + Add support for derived sub select. + Add some comments. + restore changes of settings files. * fix format issues * fix format issues * fix format issues * remove 1-element constaint when reducing list of expressions --- pom.xml | 5 + src/sqlancer/ASTBasedReducer.java | 530 ++++++++++++++++++ .../sqlancer/reducer/TestASTBasedReducer.java | 129 +++++ test/sqlancer/reducer/TestEnvironment.java | 23 +- .../reducer/TestStatementReducer.java | 6 +- 5 files changed, 688 insertions(+), 5 deletions(-) create mode 100644 src/sqlancer/ASTBasedReducer.java create mode 100644 test/sqlancer/reducer/TestASTBasedReducer.java diff --git a/pom.xml b/pom.xml index 46211aac2..4944369fa 100644 --- a/pom.xml +++ b/pom.xml @@ -363,6 +363,11 @@ commons-csv 1.9.0 + + com.github.jsqlparser + jsqlparser + 4.6 + diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java new file mode 100644 index 000000000..705d9d307 --- /dev/null +++ b/src/sqlancer/ASTBasedReducer.java @@ -0,0 +1,530 @@ +package sqlancer; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; + +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.CaseExpression; +import net.sf.jsqlparser.expression.DoubleValue; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.NullValue; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.expression.WhenClause; +import net.sf.jsqlparser.expression.operators.relational.Between; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.expression.operators.relational.InExpression; +import net.sf.jsqlparser.expression.operators.relational.ItemsList; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; +import net.sf.jsqlparser.statement.StatementVisitorAdapter; +import net.sf.jsqlparser.statement.select.GroupByElement; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectBody; +import net.sf.jsqlparser.statement.select.SelectExpressionItem; +import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; +import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.select.SubSelect; +import net.sf.jsqlparser.statement.select.WithItem; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; + +final class ExpressionTransformer { + + private static List flattenChildren(BinaryExpression expr) { + List candidates = new ArrayList<>(); + Expression lhs = expr.getLeftExpression(); + Expression rhs = expr.getRightExpression(); + candidates.add(lhs); + candidates.add(rhs); + return candidates; + } + + private static List flattenChildren(Between expr) { + Expression lhs = expr.getBetweenExpressionStart(); + Expression rhs = expr.getBetweenExpressionEnd(); + return List.of(lhs, rhs); + } + + public static List candidateExpressions(Expression expr) { + if (expr instanceof Parenthesis) { + // try removing a pair of brackets. + Parenthesis paren = (Parenthesis) expr; + return List.of(paren.getExpression()); + } else if (expr instanceof BinaryExpression) { + return flattenChildren((BinaryExpression) expr); + } else if (expr instanceof Between) { + return flattenChildren((Between) expr); + } else if (expr instanceof LongValue) { + LongValue longValue = (LongValue) expr; + if (String.valueOf(longValue).length() >= 4) { + return List.of(new NullValue(), new LongValue(10), new LongValue(0), new LongValue(1)); + } + return new ArrayList<>(); + } else if (expr instanceof DoubleValue) { + DoubleValue doubleValue = (DoubleValue) expr; + double literal = doubleValue.getValue(); + if (String.valueOf(literal).length() <= 4) { + return new ArrayList<>(); + } + double roundedValue = Math.round(literal * 10.0) / 10.0; + return List.of(new NullValue(), new DoubleValue(String.valueOf(roundedValue))); + } else if (expr instanceof StringValue) { + StringValue sv = (StringValue) expr; + String str = sv.getValue(); + if (str.length() > 4) { + return List.of(new NullValue(), new StringValue(" ")); + } + return new ArrayList<>(); + } else if (expr instanceof CaseExpression) { + CaseExpression caseExpression = (CaseExpression) expr; + return List.of(caseExpression.getSwitchExpression(), caseExpression.getElseExpression()); + } else { + return new ArrayList<>(); + } + } + + private ExpressionTransformer() throws Exception { + throw new AssertionError("Do not initialize the util class"); + } +} + +@SuppressWarnings("unchecked") +public class ASTBasedReducer, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> + implements Reducer { + + private final DatabaseProvider provider; + + @SuppressWarnings("unused") + private G state; + private G newGlobalState; + private Reproducer reproducer; + private int reduceTargetIndex; + private Statement targetStatement; + + // statement after reduction. + private List> reducedStatements; + + public ASTBasedReducer(DatabaseProvider provider) { + this.provider = provider; + } + + private void updateStatements() { + String queryString = targetStatement.toString(); + boolean couldAffectSchema = queryString.contains("CREATE TABLE") || queryString.contains("EXPLAIN"); + reducedStatements.set(reduceTargetIndex, (Query) new SQLQueryAdapter(queryString, couldAffectSchema)); + } + + public

void expressionReduce(P parent, Expression subExpr, // NOPMD + BiConsumer setter) { + boolean observeChange; + do { + observeChange = false; + List candidates = ExpressionTransformer.candidateExpressions(subExpr); + for (Expression candidate : candidates) { + try { + setter.accept(parent, candidate); + if (bugStillTriggers()) { + subExpr = candidate; + observeChange = true; + } + } catch (Exception ignoredException) { + } + } + setter.accept(parent, subExpr); + } while (observeChange); + } + + public void listElementRemovingReduce(P parent, List elms, // NOPMD + BiConsumer> setter) { + // TODO: For AST-Reducer, is delta-debugging needed ? Or just use the naive approach ? + boolean observeChange; + do { + observeChange = false; + for (int i = elms.size() - 1; i >= 0; i--) { + List reducedElms = new ArrayList<>(elms); + reducedElms.subList(i, i + 1).clear(); + setter.accept(parent, reducedElms); + try { + if (bugStillTriggers()) { + elms = reducedElms; + observeChange = true; + } + } catch (Exception e) { + System.out.println("An error occurred when trying executing reduced statements"); + e.printStackTrace(); + } + } + setter.accept(parent, elms); + } while (observeChange); + + } + + ExpressionVisitorAdapter expressionReducerVisitor = new ExpressionVisitorAdapter() { + + @Override + public void visit(InExpression expr) { + Expression rhs = expr.getRightExpression(); + + if (rhs instanceof SubSelect) { + SubSelect subSelect = (SubSelect) rhs; + subSelect.getSelectBody().accept(selectReducerVisitor); + } else { + ItemsList itemslist = expr.getRightItemsList(); + itemslist.accept(this); + } + } + + @Override + protected void visitBinaryExpression(BinaryExpression expr) { + Expression lhs = expr.getLeftExpression(); + Expression rhs = expr.getRightExpression(); + expressionReduce(expr, lhs, (expression, candidate) -> { + expression.setLeftExpression(candidate); + updateStatements(); + }); + expressionReduce(expr, rhs, (expression, candidate) -> { + expression.setRightExpression(candidate); + updateStatements(); + }); + lhs.accept(this); + rhs.accept(this); + } + + // @Override + // public void visit(DateValue value) { + // super.visit(value); + // } + // + // @Override + // public void visit(TimeValue value) { + // super.visit(value); + // } + // + // @Override + // public void visit(LikeExpression expr) { + // super.visit(expr); + // } + + @Override + public void visit(CaseExpression expr) { + Expression switchExpr = expr.getSwitchExpression(); + Expression elseExpr = expr.getElseExpression(); + + expressionReduce(expr, switchExpr, (parent, sw) -> { + parent.setSwitchExpression(sw); + updateStatements(); + }); + expressionReduce(expr, elseExpr, (parent, els) -> { + parent.setElseExpression(els); + updateStatements(); + }); + + super.visit(expr); + } + + @Override + public void visit(WhenClause whenClause) { + Expression when = whenClause.getWhenExpression(); + Expression then = whenClause.getThenExpression(); + + expressionReduce(whenClause, when, (wc, w) -> { + wc.setWhenExpression(w); + updateStatements(); + }); + + expressionReduce(whenClause, then, (wc, t) -> { + wc.setThenExpression(t); + updateStatements(); + }); + + super.visit(whenClause); + } + + @Override + public void visit(Parenthesis parenthesis) { + Expression closedExpr = parenthesis.getExpression(); + expressionReduce(parenthesis, closedExpr, (p, s) -> { + p.setExpression(s); + updateStatements(); + }); + closedExpr.accept(this); + } + + // @Override + // public void visit(Function function) { + // super.visit(function); + // } + + @Override + public void visit(ExpressionList expressionList) { + List expressions = expressionList.getExpressions(); + listElementRemovingReduce(expressionList, expressions, (l, es) -> { + l.setExpressions(es); + updateStatements(); + }); + expressions = expressionList.getExpressions(); + for (int i = 0; i < expressions.size(); i++) { + Expression expr = expressions.get(i); + int index = i; + expressionReduce(expressions, expr, (l, e) -> { + l.set(index, e); + updateStatements(); + }); + } + super.visit(expressionList); + } + + @Override + public void visit(SelectExpressionItem selectExpressionItem) { + Expression expr = selectExpressionItem.getExpression(); + expressionReduce(selectExpressionItem, expr, (item, e) -> { + item.setExpression(e); + updateStatements(); + }); + super.visit(selectExpressionItem); + } + + @Override + public void visit(SubSelect subSelect) { + subSelect.getSelectBody().accept(selectReducerVisitor); + } + + }; + + SelectVisitorAdapter selectReducerVisitor = new SelectVisitorAdapter() { + + // Clauses that would be tried removing. + // examples: + // Remove when: select * from table when 1 -> select * from table + // Remove limit: select * from table limit 1 -> select * from table + private final String[] removeList = { "Limit", "Offset", "Where", "Having", "GroupBy", "Distinct", + "OrderByElements", "Joins" }; + + // Clauses that would be tried transforming. + // examples: + // select * from t where a + b < c + d + // where clause might become one of the statement below after transformation: + // -> select * from table where c + d + // -> select * from table where a + b + + private final String[] transformList = { "Where", "Having", "FromItem", "SelectItems", "GroupBy", "Joins" }; + + // Clauses that would be visited for further reduction. + // example: + // select * from t where a + b < c + d + // Assuming that the coexistence of a and c would trigger the bug. + // The where clause : a + b < c + d would be visited and a + b, c + d would be reduced respectively. + // a + b < c + d might become a + c + private final String[] descendList = { "Where", "Having", "FromItem", "SelectItems", "GroupBy" }; + + private String getterName(String astNodeName) { + return "get" + astNodeName; + } + + private String setterName(String astNodeName) { + if (astNodeName.equals("GroupBy")) { + return "set" + astNodeName + "Element"; + } else { + return "set" + astNodeName; + } + } + + @Override + public void visit(WithItem withItem) { + // withItem.getItemsList(); + } + + @Override + public void visit(PlainSelect plainSelect) { + // transform section. Lists defined above would be iterated to get the corresponding clause name. Reflection + // is used to avoid repetitive code. e.g. The current astNodeName is When `getWhen`, `setWhen` would be + // called. + for (String astNodeName : removeList) { + try { + Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); + Object astNode = nodeGetter.invoke(plainSelect); + if (astNode == null) { + continue; + } + Method nodeSetter = plainSelect.getClass().getMethod(setterName(astNodeName), + nodeGetter.getReturnType()); + nodeSetter.invoke(plainSelect, new Object[] { null }); + updateStatements(); + if (!bugStillTriggers()) { + nodeSetter.invoke(plainSelect, astNode); + updateStatements(); + } + } catch (Exception e) { + throw new AssertionError(e); + } + } + + // Pull Up Section + for (String astNodeName : transformList) { + try { + Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); + Object astNode = nodeGetter.invoke(plainSelect); + if (astNode == null) { + continue; + } + Method nodeSetter = plainSelect.getClass().getMethod(setterName(astNodeName), + nodeGetter.getReturnType()); + + if (astNode instanceof Expression) { + Expression expr = (Expression) astNode; + expressionReduce(plainSelect, expr, (select, expression) -> { + try { + nodeSetter.invoke(select, expression); + updateStatements(); + } catch (IllegalAccessException | InvocationTargetException e) { + e.printStackTrace(); + } + }); + } else if (astNode instanceof List) { + List elms = (List) astNode; + if (elms.size() <= 1) { + continue; + } + listElementRemovingReduce(plainSelect, elms, (select, items) -> { + try { + nodeSetter.invoke(select, items); + updateStatements(); + } catch (IllegalAccessException | InvocationTargetException e) { + e.printStackTrace(); + } + }); + } else if (astNode instanceof GroupByElement) { + GroupByElement groupByElement = (GroupByElement) astNode; + ExpressionList expressionList = groupByElement.getGroupByExpressionList(); + if (expressionList == null) { + groupByElement.getGroupingSets(); + // TODO: TO BE IMPLEMENTED. + } else { + List elms = expressionList.getExpressions(); + if (elms.size() <= 1) { + continue; + } + listElementRemovingReduce(groupByElement, elms, (select, items) -> { + groupByElement.setGroupByExpressionList(new ExpressionList(items)); + updateStatements(); + }); + } + } + } catch (Exception e) { + throw new AssertionError(e); + } + } + + for (String astNodeName : descendList) { + try { + Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); + Object astNode = nodeGetter.invoke(plainSelect); + if (astNode == null) { + continue; + } + if (astNode instanceof Expression) { + ((Expression) astNode).accept(expressionReducerVisitor); + } else if (astNode instanceof List) { + // Really hacky... Some other ways to simplify it ? + List elms = (List) astNode; + for (Object obj : elms) { + if (obj instanceof SelectExpressionItem) { + ((SelectExpressionItem) obj).accept(expressionReducerVisitor); + } else if (obj instanceof Expression) { + ((Expression) obj).accept(expressionReducerVisitor); + } + } + } else if (astNode instanceof GroupByElement) { + GroupByElement groupByElement = (GroupByElement) astNode; + ExpressionList expressionList = groupByElement.getGroupByExpressionList(); + if (expressionList != null) { + expressionList.accept(expressionReducerVisitor); + } + // TODO: groupByElement.getGroupingSets() TO BE IMPLEMENTED + + } + } catch (Exception e) { + throw new AssertionError(e); + } + } + } + + @Override + public void visit(SetOperationList setOpList) { + List selectBodies = setOpList.getSelects(); + listElementRemovingReduce(setOpList, selectBodies, (optionList, selects) -> { + optionList.setSelects(selects); + updateStatements(); + }); + for (SelectBody selectBody : selectBodies) { + if (selectBody instanceof PlainSelect) { + visit((PlainSelect) selectBody); + } + } + } + + }; + + StatementVisitorAdapter statementReducerVisitor = new StatementVisitorAdapter() { + @Override + public void visit(Select select) { + SelectBody selectBody = select.getSelectBody(); + if (selectBody != null) { + selectBody.accept(selectReducerVisitor); + } + } + }; + + @Override + public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + this.state = state; + this.newGlobalState = newGlobalState; + this.reproducer = reproducer; + + List> initialBugInducingStatements = state.getState().getStatements(); + reducedStatements = new ArrayList<>(); + for (Query query : initialBugInducingStatements) { + reducedStatements.add((Query) query); + } + + for (int i = 0; i < reducedStatements.size(); i++) { + reduceTargetIndex = i; + Query query = reducedStatements.get(reduceTargetIndex); + targetStatement = CCJSqlParserUtil.parse(query.getQueryString()); + targetStatement.accept(statementReducerVisitor); + } + + newGlobalState.getState().setStatements(new ArrayList<>(reducedStatements)); + } + + private boolean bugStillTriggers() throws Exception { + try (C con2 = provider.createDatabase(newGlobalState)) { + newGlobalState.setConnection(con2); + List> candidateStatements = new ArrayList<>(reducedStatements); + newGlobalState.getState().setStatements(new ArrayList<>(candidateStatements)); + + for (Query s : candidateStatements) { + try { + s.execute(newGlobalState); + } catch (Throwable ignoredException) { + // ignore + } + } + try { + if (reproducer.bugStillTriggers(newGlobalState)) { + return true; + } + } catch (Throwable ignoredException) { + + } + } + return false; + } +} diff --git a/test/sqlancer/reducer/TestASTBasedReducer.java b/test/sqlancer/reducer/TestASTBasedReducer.java new file mode 100644 index 000000000..b65ceef21 --- /dev/null +++ b/test/sqlancer/reducer/TestASTBasedReducer.java @@ -0,0 +1,129 @@ +package sqlancer.reducer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.query.Query; + +import java.util.Arrays; +import java.util.List; + +public class TestASTBasedReducer { + @Test + void testLongStatement() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStr = { + "SELECT DISTINCT * FROM v0 WHERE ((v0.rowid || ( (v0.c + v0.d) < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)" }; + env.setInitialStatementsFromStrings(List.of(queriesStr)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("&&"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStr)); + System.out.println(reducedResult); + } + + @Test + void testReducingMultipleTokensToOne() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStr = { + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)" }; + env.setInitialStatementsFromStrings(List.of(queriesStr)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("||"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStr)); + System.out.println(reducedResult); + } + + @Test + void testMultipleStatements() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)", + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)", + "SELECT * FROM table_3;" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.toUpperCase().contains("UNION"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } + + @Test + void testJoin() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { "SELECT * FROM t0, t1, t2, t3, t4 Where t2.val = t1.val" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("t1") && queriesString.contains("WHERE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } + + @Test + void testCase() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT STRING_AGG(v0.c2) FROM t0, v0 WHERE (CASE true WHEN (ABS(true) BETWEEN (v0.c0 LIKE NULL ESCAPE v0.c2) AND (DATE '1970-01-23' NOT IN (v0.c2))) THEN (0.07914839711718646 NOT BETWEEN '' AND ((v0.c0)OR(v0.c2))) WHEN v0.c1 THEN ((v0.c1)-(v0.c0)) WHEN t0.c1 THEN (TIMESTAMP '1969-12-29 20:22:33' IN (PI(), v0.c2, (v0.c1 BETWEEN '' AND v0.rowid))) WHEN v0.c1 THEN TIMESTAMP '1969-12-16 17:24:43' WHEN ((((v0.c1)-(t0.c0)))||(t0.c0)) THEN true ELSE ((0.279978719843174)/(((v0.c1)>(DATE '1969-12-19')))) END ) GROUP BY ((DATE '1970-01-24') IS NULL), t0.c1, (CASE (v0.c1 LIKE ((0.9833120083624495)SIMILAR TO(t0.rowid)) ESCAPE CEIL(TIMESTAMP '1970-01-11 16:38:26')) WHEN t0.rowid THEN 0.27742217994251717 ELSE ((v0.c0) IS NOT NULL) END );" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.toUpperCase().contains("CASE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } + + @Test + void testConstantVar() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT STRING_AGG(v0.c2) FROM t0 GROUP BY ( (CASE (t0.rowid) WHEN t0.rowid THEN 0.27742217994251717 ELSE ((v0.c0) IS NOT NULL) END) )" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.toUpperCase().contains("CASE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } + + @Test + void testFunction() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "SELECT DATE '1970-01-11', false, t1.c1, t1.c1, (t1.c1 NOT IN (((('' LIKE t1.c2 ESCAPE t1.c1)) IS NOT NULL))) FROM t1 WHERE t1.c0 GROUP BY (((('Zlb)' IN (t1.c0)) LIKE t1.c1 ESCAPE (0.6419925594156123 BETWEEN t1.c1 AND ')-'))) ::BOOL) HAVING ((LAST_DAY(1630554083))&(AVG((CASE t1.c0 WHEN t1.c2 THEN DATE '1970-01-09' ELSE '' END )))) LIMIT 714775291;" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("AVG"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + System.out.println(Arrays.toString(queriesStrs)); + System.out.println(reducedResult); + } +} diff --git a/test/sqlancer/reducer/TestEnvironment.java b/test/sqlancer/reducer/TestEnvironment.java index 3329597aa..3ba1fc11e 100644 --- a/test/sqlancer/reducer/TestEnvironment.java +++ b/test/sqlancer/reducer/TestEnvironment.java @@ -21,8 +21,27 @@ public class TestEnvironment { private VirtualDBProvider provider = null; private VirtualDBGlobalState state, newGlobalState; - public TestEnvironment() throws Exception { + private Reducer reducer = null; + + enum ReducerType { + USING_STATEMENT_REDUCER, USING_AST_BASED_REDUCER + }; + + private TestEnvironment(ReducerType type) throws Exception { setUpTestingEnvironment(); + if (type == ReducerType.USING_STATEMENT_REDUCER) { + reducer = new StatementReducer<>(provider); + } else if (type == ReducerType.USING_AST_BASED_REDUCER) { + reducer = new ASTBasedReducer<>(provider); + } + } + + public static TestEnvironment getStatementReducerEnv() throws Exception { + return new TestEnvironment(ReducerType.USING_STATEMENT_REDUCER); + } + + public static TestEnvironment getASTBasedReducerEnv() throws Exception { + return new TestEnvironment(ReducerType.USING_AST_BASED_REDUCER); } /** @@ -99,7 +118,7 @@ public void setBugInducingCondition(Function>, Boolean> bugInducin } public void runReduce() throws Exception { - Reducer reducer = new StatementReducer<>(provider); + Reproducer reproducer = provider.generateAndTestDatabase(newGlobalState); reducer.reduce(state, reproducer, newGlobalState); } diff --git a/test/sqlancer/reducer/TestStatementReducer.java b/test/sqlancer/reducer/TestStatementReducer.java index 83c7de94e..66a9a3c4c 100644 --- a/test/sqlancer/reducer/TestStatementReducer.java +++ b/test/sqlancer/reducer/TestStatementReducer.java @@ -13,7 +13,7 @@ public class TestStatementReducer { @Test void testSimple() throws Exception { - TestEnvironment env = new TestEnvironment(); + TestEnvironment env = TestEnvironment.getStatementReducerEnv(); String[] queriesStr = { "CREATE TABLE FAKE_TABLE;", "SELECT * FROM FAKE_TABLE;", "EXIT", }; env.setInitialStatementsFromStrings(List.of(queriesStr)); @@ -30,7 +30,7 @@ void testSimple() throws Exception { @Test void testDeltaDebugging() throws Exception { - TestEnvironment env = new TestEnvironment(); + TestEnvironment env = TestEnvironment.getStatementReducerEnv(); List fakeStatements = new ArrayList<>(); for (int i = 0; i < 10000; i++) { String statement = "Statement_" + i + ";"; @@ -51,7 +51,7 @@ void testDeltaDebugging() throws Exception { @Test void testDeltaDebuggingWithStatementsCombination() throws Exception { - TestEnvironment env = new TestEnvironment(); + TestEnvironment env = TestEnvironment.getStatementReducerEnv(); List fakeStatements = new ArrayList<>(); String pattern = "(.*\\n)*(Statement_2;)\\n(.*\\n)*(Statement_318);\\n(.*\\n)*(Statement_990;)(.*\\n)*.*"; From 5bcd9ce69898f2866094535610ab0b4c04778843 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Wed, 5 Jul 2023 20:47:03 +0800 Subject: [PATCH 368/856] Feat: add TestStoneDB.java in test/sqlancer/dbms (#844) * feat: set up CI for StoneDB * feat: set up mvn -Dtest for StoneDB * fix: typo * feat: add CI entry for mvn test StoneDB * fix: change to StoneDB binary in CI * fix: change to StoneDB binary in CI * drop: manually drop CI commit * feat: add stonedb 5.7 CI * fix: rename and free space * fix: free up space again * fix: delete useless df . -H * fix: add space check and fix * fix: change to v1.0.3 * fix: typo * refactor: delete useless space check --- .github/workflows/main.yml | 29 ++++++++++++++++++++++++++++- test/sqlancer/dbms/TestStoneDB.java | 22 ++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) create mode 100644 test/sqlancer/dbms/TestStoneDB.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1bafaf3d1..990543c0d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -375,6 +375,33 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test + stonedb: + name: DBMS Tests (StoneDB) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up StoneDB + run: | + docker pull stoneatom/stonedb:v1.0.3 + docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb:v1.0.3 + docker ps + ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb:v1.0.3 --format "{{.ID}}") + docker exec $ContainerID bash + sleep 180s + docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + - name: Set up JDK 11 + uses: actions/setup-java@v1.4.4 + with: + java-version: 11 + - name: Build + run: mvn -B package -DskipTests=true + - name: StoneDB Tests + run: | + STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDB + tidb: name: DBMS Tests (TiDB) runs-on: ubuntu-latest @@ -550,4 +577,4 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - mvn -Dtest=TestStatementReducer test + mvn -Dtest=TestStatementReducer test \ No newline at end of file diff --git a/test/sqlancer/dbms/TestStoneDB.java b/test/sqlancer/dbms/TestStoneDB.java new file mode 100644 index 000000000..14de121be --- /dev/null +++ b/test/sqlancer/dbms/TestStoneDB.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestStoneDB { + + String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); + boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); + + @Test + public void testStoneDB() { + assumeTrue(stoneDBIsAvailable); + assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", + "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "NoREC")); + } + +} From 7f99a129ad4d6ef60816bfc62c2e38d350a4a337 Mon Sep 17 00:00:00 2001 From: Nathee Jaywaree <73115539+eltfshr@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:08:50 +0800 Subject: [PATCH 369/856] feat: add MySQLUpdateGenerator class (#849) * Add MySQLUpdateGenerator class * Extract common update insert errors in mysql --- src/sqlancer/mysql/MySQLErrors.java | 11 ++++ src/sqlancer/mysql/MySQLProvider.java | 5 ++ .../mysql/gen/MySQLInsertGenerator.java | 10 +--- .../mysql/gen/MySQLUpdateGenerator.java | 58 +++++++++++++++++++ 4 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/mysql/gen/MySQLUpdateGenerator.java diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 540a5c5db..9a8f076ce 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -16,4 +16,15 @@ public static void addExpressionErrors(ExpectedErrors errors) { } } + public static void addInsertUpdateErrors(ExpectedErrors errors) { + errors.add("doesn't have a default value"); + errors.add("Data truncation"); + errors.add("Incorrect integer value"); + errors.add("Duplicate entry"); + errors.add("Data truncated for column"); + errors.add("Data truncated for functional index"); + errors.add("cannot be null"); + errors.add("Incorrect decimal value"); + } + } diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 201e0c799..9392550cc 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -31,6 +31,7 @@ import sqlancer.mysql.gen.MySQLSetGenerator; import sqlancer.mysql.gen.MySQLTableGenerator; import sqlancer.mysql.gen.MySQLTruncateTableGenerator; +import sqlancer.mysql.gen.MySQLUpdateGenerator; import sqlancer.mysql.gen.admin.MySQLFlush; import sqlancer.mysql.gen.admin.MySQLReset; import sqlancer.mysql.gen.datadef.MySQLIndexGenerator; @@ -67,6 +68,7 @@ enum Action implements AbstractAction { String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); return MySQLTableGenerator.generate(g, tableName); }), // + UPDATE(MySQLUpdateGenerator::create), // DELETE(MySQLDeleteGenerator::delete), // DROP_INDEX(MySQLDropIndex::generate); @@ -132,6 +134,9 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { case SELECT_INFO: nrPerformed = r.getInteger(0, 10); break; + case UPDATE: + nrPerformed = r.getInteger(0, 10); + break; case DELETE: nrPerformed = r.getInteger(0, 10); break; diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 14994a140..86083fd2d 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -7,6 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -88,14 +89,7 @@ private SQLQueryAdapter generateInto() { } sb.append(")"); } - errors.add("doesn't have a default value"); - errors.add("Data truncation"); - errors.add("Incorrect integer value"); - errors.add("Duplicate entry"); - errors.add("Data truncated for functional index"); - errors.add("Data truncated for column"); - errors.add("cannot be null"); - errors.add("Incorrect decimal value"); + MySQLErrors.addInsertUpdateErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java new file mode 100644 index 000000000..55ba3dd45 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java @@ -0,0 +1,58 @@ +package sqlancer.mysql.gen; + +import java.sql.SQLException; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; + +public class MySQLUpdateGenerator extends AbstractUpdateGenerator { + + private final MySQLGlobalState globalState; + private MySQLExpressionGenerator gen; + + public MySQLUpdateGenerator(MySQLGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter create(MySQLGlobalState globalState) throws SQLException { + return new MySQLUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() throws SQLException { + MySQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new MySQLExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + MySQLErrors.addExpressionErrors(errors); + sb.append(MySQLVisitor.asString(gen.generateExpression())); + } + MySQLErrors.addInsertUpdateErrors(errors); + errors.add("doesn't have this option"); + + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void updateValue(MySQLColumn column) { + if (Randomly.getBoolean()) { + sb.append(gen.generateConstant()); + } else if (Randomly.getBoolean()) { + sb.append("DEFAULT"); + } else { + sb.append(MySQLVisitor.asString(gen.generateExpression())); + } + } + +} From 26de30200a03b6ed13b16f4c548d9783aa78cd5c Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Fri, 7 Jul 2023 20:32:50 +0800 Subject: [PATCH 370/856] fix: SQL syntax errors and add expected errors in StoneDB implementation Part 1 (#846) * fix: add space when creating table * fix: add expected errors and key length specification * feat: generate comments randomly * fix: add expected errors and key restriction * feat: add support for more map actions * fix: add space between INDEX and indexName * fix: add space and {} * fix: remove pre-added errors before testing * refactor: rename delete useless as when deleting from table * fix: add missing space * fix: add missing ( * fix: add missing length for varbinary type * fix: string generation strategy in comments * fix: add provider * fix: use global randomly * fix: insert one row * feat: add expected errors * feat: no length * feat: no length * style: format * fix: Tianmu engine does not support fulltext index. * fix: fix alter table and getwithoutnull * fix: add expected errors * feat: support StringVisitor of StoneSelect * fix: delete useless import * fix: syntax error in index create * style: run formate * fix: syntax errors * fix: delete AS in DELETE * feat: add expectedErrors for all generator * refactor: change cast operation to lambda and fix dataType toString error * feat: add expected errors * refactor: remove redundant identifier * refactor: remove unused expected errors * refactor: use parent StringBuilder * refactor: add more expected errors * refactor: change action perform times * refactor: rename and change action perform times * fix: delete unexcepted errors * refactor: delete useless code --- src/sqlancer/Main.java | 2 + src/sqlancer/stonedb/StoneDBProvider.java | 44 ++++++-- src/sqlancer/stonedb/StoneDBSchema.java | 7 ++ .../stonedb/StoneDBToStringVisitor.java | 62 ++++++++++ .../gen/StoneDBExpressionGenerator.java | 11 +- .../gen/StoneDBIndexCreateGenerator.java | 106 ++++++++++++++++++ .../stonedb/gen/StoneDBIndexGenerator.java | 75 ------------- .../gen/StoneDBTableAlterGenerator.java | 19 +++- .../gen/StoneDBTableCreateGenerator.java | 64 ++++++----- ....java => StoneDBTableDeleteGenerator.java} | 19 ++-- ....java => StoneDBTableInsertGenerator.java} | 25 ++++- .../stonedb/oracle/StoneDBNoRECOracle.java | 3 +- 12 files changed, 290 insertions(+), 147 deletions(-) create mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java rename src/sqlancer/stonedb/gen/{StoneDBDeleteGenerator.java => StoneDBTableDeleteGenerator.java} (77%) rename src/sqlancer/stonedb/gen/{StoneDBInsertGenerator.java => StoneDBTableInsertGenerator.java} (73%) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index dfdc949d6..76d7a36f4 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -45,6 +45,7 @@ import sqlancer.postgres.PostgresProvider; import sqlancer.questdb.QuestDBProvider; import sqlancer.sqlite3.SQLite3Provider; +import sqlancer.stonedb.StoneDBProvider; import sqlancer.tidb.TiDBProvider; import sqlancer.timescaledb.TimescaleDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; @@ -660,6 +661,7 @@ private static void checkForIssue799(List> providers) providers.add(new PostgresProvider()); providers.add(new QuestDBProvider()); providers.add(new SQLite3Provider()); + providers.add(new StoneDBProvider()); providers.add(new TiDBProvider()); providers.add(new TimescaleDBProvider()); providers.add(new YCQLProvider()); diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java index 376235d92..e0c317c53 100644 --- a/src/sqlancer/stonedb/StoneDBProvider.java +++ b/src/sqlancer/stonedb/StoneDBProvider.java @@ -19,11 +19,13 @@ import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; -import sqlancer.stonedb.gen.StoneDBDeleteGenerator; -import sqlancer.stonedb.gen.StoneDBIndexGenerator; -import sqlancer.stonedb.gen.StoneDBInsertGenerator; +import sqlancer.stonedb.gen.StoneDBIndexCreateGenerator; +import sqlancer.stonedb.gen.StoneDBIndexDropGenerator; import sqlancer.stonedb.gen.StoneDBTableAlterGenerator; import sqlancer.stonedb.gen.StoneDBTableCreateGenerator; +import sqlancer.stonedb.gen.StoneDBTableDeleteGenerator; +import sqlancer.stonedb.gen.StoneDBTableInsertGenerator; +import sqlancer.stonedb.gen.StoneDBTableUpdateGenerator; @AutoService(DatabaseProvider.class) public class StoneDBProvider extends SQLProviderAdapter { @@ -40,11 +42,17 @@ protected StoneDBSchema readSchema() throws Exception { } enum Action implements AbstractAction { - SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES")), // - ALTER_TABLE(StoneDBTableAlterGenerator::generate), // - DELETE(StoneDBDeleteGenerator::generate), // - INDEX(StoneDBIndexGenerator::generate), // - INSERT(StoneDBInsertGenerator::generate); // + TABLES_SHOW((g) -> new SQLQueryAdapter("SHOW TABLES")), // + TABLE_ALTER(StoneDBTableAlterGenerator::generate), // + TABLE_CREATE((g) -> { + String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); + return StoneDBTableCreateGenerator.generate(g, tableName); + }), // + TABLE_DELETE(StoneDBTableDeleteGenerator::generate), // + TABLE_UPDATE(StoneDBTableUpdateGenerator::generate), // + INDEX_CREATE(StoneDBIndexCreateGenerator::generate), // + INDEX_DROP(StoneDBIndexDropGenerator::generate), // + TABLE_INSERT(StoneDBTableInsertGenerator::generate); // private final SQLQueryProvider sqlQueryProvider; @@ -59,10 +67,24 @@ public Query getQuery(StoneDBGlobalState globalState) throws Exception { } private static int mapActions(StoneDBGlobalState globalState, Action a) { - globalState.getRandomly(); + Randomly r = globalState.getRandomly(); switch (a) { - case SHOW_TABLES: - return 1; + case TABLES_SHOW: + return r.getInteger(0, 1); + case TABLE_ALTER: + return r.getInteger(0, 5); + case TABLE_CREATE: + return r.getInteger(0, 1); + case TABLE_DELETE: + return r.getInteger(0, 10); + case TABLE_INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + case TABLE_UPDATE: + return r.getInteger(0, 1); + case INDEX_CREATE: + return r.getInteger(0, 1); + case INDEX_DROP: + return r.getInteger(0, 1); default: throw new AssertionError(a); } diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java index 58170ce73..c24b67793 100644 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -5,6 +5,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import sqlancer.Randomly; import sqlancer.SQLConnection; @@ -25,6 +27,11 @@ public enum StoneDBDataType { CHAR, VARCHAR, TINYTEXT, TEXT, MEDIUMTEXT, LONGTEXT, BINARY, VARBINARY, TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB; public static StoneDBDataType getRandomWithoutNull() { + List collect = Stream.of(values()).filter(c -> c != NULL).collect(Collectors.toList()); + return Randomly.fromList(collect); + } + + public static StoneDBDataType getRandom() { return Randomly.fromOptions(values()); } diff --git a/src/sqlancer/stonedb/StoneDBToStringVisitor.java b/src/sqlancer/stonedb/StoneDBToStringVisitor.java index 3cebd97f1..9d7711873 100644 --- a/src/sqlancer/stonedb/StoneDBToStringVisitor.java +++ b/src/sqlancer/stonedb/StoneDBToStringVisitor.java @@ -4,12 +4,18 @@ import sqlancer.common.ast.newast.Node; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; public class StoneDBToStringVisitor extends NewToStringVisitor { @Override public void visitSpecific(Node expr) { if (expr instanceof StoneDBConstant) { visit((StoneDBConstant) expr); + } else if (expr instanceof StoneDBSelect) { + visit((StoneDBSelect) expr); + } else if (expr instanceof StoneDBJoin) { + visit((StoneDBJoin) expr); } else { throw new AssertionError(expr.getClass()); } @@ -19,6 +25,62 @@ private void visit(StoneDBConstant constant) { sb.append(constant.toString()); } + private void visit(StoneDBSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + private void visit(StoneDBJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getNaturalJoinType() != null) { + sb.append(join.getNaturalJoinType()); + } + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + public static String asString(Node expr) { StoneDBToStringVisitor visitor = new StoneDBToStringVisitor(); visitor.visit(expr); diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 59fb7e052..385154815 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -18,7 +18,6 @@ import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -38,14 +37,8 @@ private enum Expression { public static class StoneDBCastOperation extends NewUnaryPostfixOperatorNode { - public StoneDBCastOperation(Node expr, StoneDBCompositeDataType type) { - super(expr, new Operator() { - - @Override - public String getTextRepresentation() { - return "::" + type.toString(); - } - }); + public StoneDBCastOperation(Node expr, StoneDBDataType type) { + super(expr, () -> "::" + type.toString()); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java new file mode 100644 index 000000000..3b7e63422 --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java @@ -0,0 +1,106 @@ +package sqlancer.stonedb.gen; + +import sqlancer.Randomly; +import sqlancer.Randomly.StringGenerationStrategy; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; + +public class StoneDBIndexCreateGenerator { + private final StoneDBGlobalState globalState; + private final Randomly r; + // which table to add index + StoneDBTable table; + private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBIndexCreateGenerator(StoneDBGlobalState globalState) { + this.globalState = globalState; + r = globalState.getRandomly(); + table = globalState.getSchema().getRandomTable(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { + return new StoneDBIndexCreateGenerator(globalState).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("CREATE "); + sb.append(Randomly.fromOptions("UNIQUE", "FULLTEXT", "SPATIAL")); + sb.append(" INDEX "); + sb.append(globalState.getSchema().getFreeIndexName()); + appendIndexType(); + sb.append(" ON "); + sb.append(table.getName()); + appendKeyPart(); + appendIndexOption(); + appendAlgoOrLockOption(); + addExpectedErrors(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void addExpectedErrors() { + // java.sql.SQLException: Tianmu engine does not support unique index. + errors.add("Tianmu engine does not support unique index"); + // java.sql.SQLException: Tianmu engine does not support fulltext index. + errors.add("Tianmu engine does not support fulltext index"); + // java.sql.SQLException: The used table type doesn't support SPATIAL indexes + errors.add("The used table type doesn't support SPATIAL indexes"); + // java.sql.SQLException: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY. + errors.add("ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY."); + // java.sql.SQLSyntaxErrorException: Key column 'c0' doesn't exist in table + errors.add("doesn't exist in table"); + // java.sql.SQLSyntaxErrorException: A SPATIAL index may only contain a geometrical type column + errors.add("A SPATIAL index may only contain a geometrical type column"); + } + + private void appendIndexType() { + // The index_type clause cannot be used for FULLTEXT INDEX or SPATIAL INDEX specifications. + if (sb.toString().contains("FULLTEXT") || sb.toString().contains("SPATIAL")) { + return; + } + if (Randomly.getBoolean()) { + return; + } + sb.append(" USING "); + sb.append(Randomly.fromOptions("BTREE", "HASH")); + } + + private void appendKeyPart() { + sb.append("("); + sb.append(table.getRandomColumn().getName()); + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" ASC", " DESC")); + } + sb.append(")"); + } + + private void appendIndexOption() { + if (Randomly.getBoolean()) { + return; + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions("KEY_BLOCK_SIZE ", "KEY_BLOCK_SIZE = ")); + sb.append(r.getInteger(1, Math.max(1, Randomly.smallNumber()))); + sb.append(" "); + } + if (Randomly.getBoolean()) { + StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; + sb.append(String.format("COMMENT '%s' ", strategy.getString(r))); + } + } + + private void appendAlgoOrLockOption() { + if (Randomly.getBoolean()) { + return; + } + if (Randomly.getBoolean()) { + sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); + sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); + } else { + sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); + sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); + } + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java deleted file mode 100644 index 9a86cf7e4..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBIndexGenerator.java +++ /dev/null @@ -1,75 +0,0 @@ -package sqlancer.stonedb.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; - -public class StoneDBIndexGenerator { - private final StoneDBGlobalState globalState; - // which table to add index - StoneDBTable table; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBIndexGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - table = globalState.getSchema().getRandomTable(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBIndexGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("CREATE "); - sb.append(Randomly.fromOptions("UNIQUE", "FULLTEXT", "SPATIAL")); - sb.append(" INDEX"); - sb.append(globalState.getSchema().getFreeIndexName()); - appendIndexType(); - sb.append(" ON "); - sb.append(table.getName()); - appendKeyPart(); - appendIndexOption(); - appendAlgoOrLockOption(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void appendIndexType() { - if (Randomly.getBoolean()) { - return; - } - sb.append(" USING "); - sb.append(Randomly.fromOptions("BTREE", "HASH")); - } - - private void appendKeyPart() { - sb.append("("); - sb.append(table.getRandomColumn().getName()); - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions("ASC", "DESC")); - } - sb.append(")"); - } - - private void appendIndexOption() { - if (Randomly.getBoolean()) { - return; - } - sb.append(Randomly.fromOptions(" VISIBLE", " INVISIBLE")); - } - - private void appendAlgoOrLockOption() { - if (Randomly.getBoolean()) { - return; - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); - sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); - } else { - sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); - sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index 55ebd0715..f25174948 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -4,6 +4,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; @@ -38,7 +39,7 @@ private SQLQueryAdapter getQuery() { String columnName = table.getFreeColumnName(); sb.append(columnName); sb.append(" "); - sb.append(StoneDBCompositeDataType.getRandomWithoutNull()); + sb.append(StoneDBCompositeDataType.getRandomWithoutNull().getPrimitiveDataType().toString()); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); @@ -54,9 +55,11 @@ private SQLQueryAdapter getQuery() { break; case ALTER_COLUMN: sb.append(Randomly.fromOptions("ALTER COLUMN ", "ALTER ")); - sb.append(table.getRandomColumn().getName()); + StoneDBColumn randomColumn = table.getRandomColumn(); + sb.append(randomColumn.getName()); if (Randomly.getBoolean()) { - sb.append(" SET DEFAULT ").append(generator.generateExpression()); + sb.append(" SET DEFAULT ").append(generator + .generateConstant(randomColumn.getType().getPrimitiveDataType(), Randomly.getBoolean())); } else { sb.append(" DROP DEFAULT"); } @@ -70,7 +73,7 @@ private SQLQueryAdapter getQuery() { String newColumnName = table.getFreeColumnName(); sb.append(oldColumnName).append(" ").append(newColumnName); sb.append(" "); - sb.append(StoneDBCompositeDataType.getRandomWithoutNull()); + sb.append(StoneDBCompositeDataType.getRandomWithoutNull().getPrimitiveDataType().toString()); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); @@ -81,7 +84,7 @@ private SQLQueryAdapter getQuery() { } break; case RENAME_COLUMN: - sb.append("RENAME COLUMN"); + sb.append("RENAME COLUMN "); sb.append(table.getRandomColumn().getName()); sb.append(" TO "); sb.append(table.getFreeColumnName()); @@ -89,6 +92,12 @@ private SQLQueryAdapter getQuery() { default: throw new AssertionError(action); } + addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors, true); } + + private void addExpectedErrors() { + // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead + errors.add("You can't delete all columns with ALTER TABLE; use DROP TABLE instead"); + } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java index 134e0edfe..622c86c3c 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java @@ -1,11 +1,15 @@ package sqlancer.stonedb.gen; +import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.PRIMARY_KEY; +import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.UNIQUE; + import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import sqlancer.Randomly; +import sqlancer.Randomly.StringGenerationStrategy; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -22,6 +26,7 @@ public class StoneDBTableCreateGenerator { private final boolean allowPrimaryKey; private boolean setPrimaryKey; private final StringBuilder sb = new StringBuilder(); + ExpectedErrors errors = new ExpectedErrors(); private final Randomly r; public StoneDBTableCreateGenerator(StoneDBGlobalState globalState, String tableName) { @@ -36,24 +41,36 @@ public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String ta } public SQLQueryAdapter getQuery() { - ExpectedErrors errors = new ExpectedErrors(); - sb.append(Randomly.fromOptions("CREATE TABLE", "CREATE TEMPORARY TABLE")); + sb.append(Randomly.fromOptions("CREATE TABLE ", "CREATE TEMPORARY TABLE ")); if (Randomly.getBoolean()) { - sb.append(" IF NOT EXISTS "); + sb.append("IF NOT EXISTS "); } sb.append(tableName); // ues link statement if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { sb.append(" LIKE "); sb.append(schema.getRandomTable().getName()); - return new SQLQueryAdapter(sb.toString(), true); } else { appendColumns(); sb.append(" "); appendTableOptions(); - addCommonErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true); } + addExpectedErrors(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void addExpectedErrors() { + // java.sql.SQLSyntaxErrorException: BLOB/TEXT column 'c0' used in key specification without a key length + errors.add("used in key specification without a key length"); + // java.sql.SQLException: Tianmu engine does not support unique index. + errors.add("Tianmu engine does not support unique index"); + // java.sql.SQLException: BLOB column 'c0' can't be used in key specification with the used table type + errors.add("can't be used in key specification with the used table type"); + // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes + errors.add("Specified key was too long; max key length is 3072 bytes"); + // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT + // instead + errors.add("Column length too big for column"); } private enum TableOptions { @@ -144,18 +161,6 @@ private void appendTableOptions() { } } - private void addCommonErrors(ExpectedErrors list) { - list.add("The storage engine for the table doesn't support"); - list.add("doesn't have this option"); - list.add("must include all columns"); - list.add("not allowed type for this type of partitioning"); - list.add("doesn't support BLOB/TEXT columns"); - list.add("A BLOB field is not allowed in partition function"); - list.add("Too many keys specified; max 1 keys allowed"); - list.add("The total length of the partitioning fields is too large"); - list.add("Got error -1 - 'Unknown error -1' from storage engine"); - } - private void appendColumns() { sb.append("("); for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { @@ -178,11 +183,10 @@ private void appendColumnDefinition() { sb.append(" "); StoneDBDataType randomType = StoneDBDataType.getRandomWithoutNull(); appendType(randomType); - sb.append(" "); appendColumnOption(randomType); } - private enum ColumnOptions { + protected enum ColumnOptions { NULL_OR_NOT_NULL, UNIQUE, COMMENT, COLUMN_FORMAT, STORAGE, PRIMARY_KEY } @@ -195,10 +199,14 @@ private void appendColumnOption(StoneDBDataType type) { // if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { // tableHasNullableColumn = true; // } + // only use one key, unique key or primary key, but not both + if (columnOptions.contains(PRIMARY_KEY) && columnOptions.contains(UNIQUE)) { + columnOptions.remove(Randomly.fromOptions(PRIMARY_KEY, UNIQUE)); + } if (isTextType) { // TODO: restriction due to the limited key length - columnOptions.remove(ColumnOptions.PRIMARY_KEY); - columnOptions.remove(ColumnOptions.UNIQUE); + columnOptions.remove(PRIMARY_KEY); + columnOptions.remove(UNIQUE); } for (ColumnOptions o : columnOptions) { sb.append(" "); @@ -222,8 +230,8 @@ private void appendColumnOption(StoneDBDataType type) { } break; case COMMENT: - // TODO: generate randomly - sb.append(String.format("COMMENT '%s' ", "asdf")); + StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; + sb.append(String.format("COMMENT '%s' ", strategy.getString(r))); break; case COLUMN_FORMAT: sb.append("COLUMN_FORMAT "); @@ -297,12 +305,10 @@ private void appendType(StoneDBDataType randomType) { sb.append("TIMESTAMP"); break; case CHAR: - sb.append("CHAR"); + sb.append("CHAR").append(Randomly.fromOptions("", "(" + r.getInteger(0, 255) + ")")); break; case VARCHAR: - sb.append("VARCHAR("); - sb.append(r.getInteger(0, 65535)); - sb.append(")"); + sb.append("VARCHAR").append("(").append(r.getInteger(0, 65535)).append(")"); break; case TINYTEXT: sb.append("TINYTEXT"); @@ -320,7 +326,7 @@ private void appendType(StoneDBDataType randomType) { sb.append("BINARY"); break; case VARBINARY: - sb.append("VARBINARY"); + sb.append("VARBINARY").append("(").append(r.getInteger(0, 65535)).append(")"); break; case TINYBLOB: sb.append("TINYBLOB"); diff --git a/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java similarity index 77% rename from src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java rename to src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index d52aed526..8e6c89dee 100644 --- a/src/sqlancer/stonedb/gen/StoneDBDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -10,17 +10,19 @@ import sqlancer.stonedb.StoneDBSchema.StoneDBTable; import sqlancer.stonedb.StoneDBToStringVisitor; -public final class StoneDBDeleteGenerator { +public final class StoneDBTableDeleteGenerator { private final StoneDBGlobalState globalState; private final StringBuilder sb = new StringBuilder(); ExpectedErrors errors = new ExpectedErrors(); + Randomly r; - private StoneDBDeleteGenerator(StoneDBGlobalState globalState) { + private StoneDBTableDeleteGenerator(StoneDBGlobalState globalState) { this.globalState = globalState; + r = globalState.getRandomly(); } public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBDeleteGenerator(globalState).getQuery(); + return new StoneDBTableDeleteGenerator(globalState).getQuery(); } public SQLQueryAdapter getQuery() { @@ -37,10 +39,6 @@ public SQLQueryAdapter getQuery() { } sb.append(" FROM "); sb.append(randomTable.getName()); - if (Randomly.getBoolean()) { - sb.append(" AS "); - sb.append(globalState.getSchema().getRandomTable().getName()); - } if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) @@ -48,12 +46,13 @@ public SQLQueryAdapter getQuery() { } if (Randomly.getBoolean()) { sb.append(" ORDER BY "); - sb.append(Randomly.fromOptions( - randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))); + sb.append(String.join(", ", Randomly.fromOptions( + randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))) + .replace('[', '(').replace(']', ')')); } if (Randomly.getBoolean()) { sb.append(" LIMIT "); - sb.append(new Randomly().getInteger()); + sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); } return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java similarity index 73% rename from src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java rename to src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index 05a2a24e9..7a0e3847f 100644 --- a/src/sqlancer/stonedb/gen/StoneDBInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -13,23 +13,22 @@ import sqlancer.stonedb.StoneDBSchema.StoneDBTable; import sqlancer.stonedb.StoneDBToStringVisitor; -public class StoneDBInsertGenerator extends AbstractInsertGenerator { +public class StoneDBTableInsertGenerator extends AbstractInsertGenerator { private final StoneDBGlobalState globalState; // which table to insert into private final StoneDBTable table; // which subset columns of the table to add values private final List columns; - private final StringBuilder sb = new StringBuilder(); ExpectedErrors errors = new ExpectedErrors(); - public StoneDBInsertGenerator(StoneDBGlobalState globalState) { + public StoneDBTableInsertGenerator(StoneDBGlobalState globalState) { this.globalState = globalState; table = globalState.getSchema().getRandomTable(); columns = table.getRandomNonEmptyColumnSubset(); } public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBInsertGenerator(globalState).getQuery(); + return new StoneDBTableInsertGenerator(globalState).getQuery(); } private SQLQueryAdapter getQuery() { @@ -45,9 +44,21 @@ private SQLQueryAdapter getQuery() { sb.append(table.getName()); appendPartition(); appendColumnsAndValues(columns); + addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } + private void addExpectedErrors() { + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row + errors.add("Data truncation: Out of range value for column "); + // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 'field list' + errors.add("Unknown column "); + // java.sql.SQLException: Insert duplicate key on row: 4, pk: 138609795916627968 + errors.add("Insert duplicate key on row: "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: + errors.add("Data truncation: Incorrect datetime value: "); + } + private void appendPartition() { } @@ -60,6 +71,7 @@ private void appendColumnsAndValues(List columns) { appendValues(); } + // append nrRows rows private void appendValues() { int nrRows; if (Randomly.getBoolean()) { @@ -72,6 +84,7 @@ private void appendValues() { } } + // append all columns of one row private void appendOneValue(int nrRow) { if (nrRow != 0) { sb.append(", "); @@ -81,12 +94,12 @@ private void appendOneValue(int nrRow) { if (c != 0) { sb.append(", "); } - sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState).generateConstant())); - + insertValue(columns.get(c)); } sb.append(")"); } + // append one column of one row @Override protected void insertValue(StoneDBColumn column) { if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 74ed75a91..1d3dac312 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -21,7 +21,6 @@ import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; import sqlancer.stonedb.StoneDBSchema.StoneDBTables; @@ -69,7 +68,7 @@ private int getUnoptimizedQueryCount(List> tableList, Node asText = new NewPostfixTextNode<>(new StoneDBCastOperation( new NewPostfixTextNode<>(randomWhereCondition, " IS NOT NULL AND " + StoneDBToStringVisitor.asString(randomWhereCondition)), - new StoneDBCompositeDataType(StoneDBDataType.INT, 8)), "as count"); + StoneDBDataType.INT), "as count"); select.setFetchColumns(List.of(asText)); select.setFromList(tableList); select.setJoinList(joins); From e3fbde0df29b386f100c371b1fcf11ff7005c7c6 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Mon, 10 Jul 2023 15:52:20 +0800 Subject: [PATCH 371/856] feat: add expected errors (#852) --- src/sqlancer/stonedb/StoneDBBugs.java | 3 +++ src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java index 7ed539763..01040c429 100644 --- a/src/sqlancer/stonedb/StoneDBBugs.java +++ b/src/sqlancer/stonedb/StoneDBBugs.java @@ -1,6 +1,9 @@ package sqlancer.stonedb; public final class StoneDBBugs { + // https://github.com/stoneatom/stonedb/issues/1933 + public static boolean bug1933 = true; + private StoneDBBugs() { } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index 8e6c89dee..3ff809740 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -6,6 +6,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.stonedb.StoneDBBugs; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; import sqlancer.stonedb.StoneDBToStringVisitor; @@ -54,6 +55,13 @@ public SQLQueryAdapter getQuery() { sb.append(" LIMIT "); sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); } + addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } + + private void addExpectedErrors() { + if (StoneDBBugs.bug1933) { + errors.add("assert failed on i < m_idx.size() at tianmu_attr.h:387, msg: [bad dpn index 0/0]"); + } + } } From 517e39474b609d9c582c9f8266e2a3f8568f3070 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Wed, 12 Jul 2023 22:13:22 +0800 Subject: [PATCH 372/856] fix & refactor: fix StoneDB implementation bugs and refactor code, Part 2 (#854) * fix: add space when creating table * fix: add expected errors and key length specification * feat: generate comments randomly * fix: add expected errors and key restriction * feat: add support for more map actions * fix: add space between INDEX and indexName * fix: add space and {} * fix: remove pre-added errors before testing * refactor: rename delete useless as when deleting from table * fix: add missing space * fix: add missing ( * fix: add missing length for varbinary type * fix: string generation strategy in comments * fix: add provider * fix: use global randomly * fix: insert one row * feat: add expected errors * feat: no length * feat: no length * style: format * fix: Tianmu engine does not support fulltext index. * fix: fix alter table and getwithoutnull * fix: add expected errors * feat: support StringVisitor of StoneSelect * fix: delete useless import * fix: syntax error in index create * style: run formate * fix: syntax errors * fix: delete AS in DELETE * feat: add expectedErrors for all generator * refactor: change cast operation to lambda and fix dataType toString error * feat: add expected errors * refactor: remove redundant identifier * refactor: remove unused expected errors * refactor: use parent StringBuilder * refactor: add more expected errors * feat: add some options according to docs * refactor: change action perform times * refactor: rename and change action perform times * fix: delete unexcepted errors * feat: add one expected error * feat: add another option when assigning assignment list * refactor: refactor append type, reused function * refactor: refactor append type argus * fix: fix null pointer exception * refactor: delete useless code * fix: fix cast operation bug in StoneDB * feat: add some options according to docs * feat: add one expected error * feat: add another option when assigning assignment list * fix: fix null pointer exception * fix: fix cast operation bug in StoneDB * fix: fix cast operation bug in StoneDB * fix: Tianmu engine does not support fulltext index. * feat: add expectedErrors for all generator * refactor: remove unused expected errors * refactor: delete useless code * feat: add some options according to docs * feat: add one expected error * feat: add another option when assigning assignment list * fix: fix null pointer exception * fix: fix cast operation bug in StoneDB * fix: fix cast operation bug in StoneDB * fix: delete rename column * feat: add expected errors * refactor: reuse code * refactor: refactor append type argus * refactor: reuse code, rename, fix * fix: syntax error * fix: change / alter column error * fix: change / alter column error * fix: change / alter column error * fix: duplicate data type when add column in alter table * fix: duplicate data type when add column in alter table * fix: no space error * fix: fix as count in NoREC Oracle * fix: fix as count in NoREC Oracle * fix: fix as count in NoREC Oracle * fix: fix bitwise operation * fix: fix binary operation * fix: fix in operation * feat: add expected errors * feat: add expected errors * feat: fix errors in fetch size * feat: add expected errors * fix: regex match string * feat: add expected errors * git: merge main * feat: disable delete order by * feat: disable delete order by * feat: disable delete order by --- src/sqlancer/stonedb/StoneDBSchema.java | 81 +++++++++++ .../stonedb/StoneDBToStringVisitor.java | 12 ++ .../gen/StoneDBExpressionGenerator.java | 53 ++++++-- .../gen/StoneDBIndexCreateGenerator.java | 20 ++- .../gen/StoneDBTableAlterGenerator.java | 85 ++++++++---- .../gen/StoneDBTableCreateGenerator.java | 127 +++--------------- .../gen/StoneDBTableDeleteGenerator.java | 9 +- .../gen/StoneDBTableInsertGenerator.java | 5 + .../gen/StoneDBTableUpdateGenerator.java | 7 +- .../stonedb/oracle/StoneDBNoRECOracle.java | 20 ++- 10 files changed, 245 insertions(+), 174 deletions(-) diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java index c24b67793..2150cbba1 100644 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -92,6 +92,87 @@ public static Node getRandomValue(StoneDBDataType dataType) { } return null; } + + public static String getTypeAndValue(StoneDBDataType dataType) { + StringBuilder sb = new StringBuilder(); + switch (dataType) { + case TINYINT: + return "TINYINT"; + case SMALLINT: + return "SMALLINT"; + case MEDIUMINT: + return "MEDIUMINT"; + case INT: + return "INT"; + case BIGINT: + return "BIGINT"; + case FLOAT: + sb.append("FLOAT"); + optionallyAddPrecisionAndScale(sb); + return sb.toString(); + case DOUBLE: + sb.append("DOUBLE"); + optionallyAddPrecisionAndScale(sb); + return sb.toString(); + case DECIMAL: + return "DECIMAL"; // The default value is P(10,0); + case YEAR: + return "YEAR"; + case TIME: + return "TIME"; + case DATE: + return "DATE"; + case DATETIME: + return "DATETIME"; + case TIMESTAMP: + return "TIMESTAMP"; + case CHAR: + sb.append("CHAR").append(Randomly.fromOptions("", "(" + new Randomly().getInteger(0, 255) + ")")); + return sb.toString(); + case VARCHAR: + sb.append("VARCHAR").append("(").append(new Randomly().getInteger(0, 65535)).append(")"); + return sb.toString(); + case TINYTEXT: + return "TINYTEXT"; + case TEXT: + return "TEXT"; + case MEDIUMTEXT: + return "MEDIUMTEXT"; + case LONGTEXT: + return "LONGTEXT"; + case BINARY: + return "BINARY"; + case VARBINARY: + sb.append("VARBINARY").append("(").append(new Randomly().getInteger(0, 65535)).append(")"); + return sb.toString(); + case TINYBLOB: + return "TINYBLOB"; + case BLOB: + return "BLOB"; + case MEDIUMBLOB: + return "MEDIUMBLOB"; + case LONGBLOB: + return "LONGBLOB"; + default: + throw new AssertionError(); + } + } + + private static void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean()) { + sb.append("("); + // The maximum number of digits (M) for DECIMAL is 65 + long m = Randomly.getNotCachedInteger(1, 65); + sb.append(m); + sb.append(", "); + // The maximum number of supported decimals (D) is 30 + long nCandidate = Randomly.getNotCachedInteger(1, 30); + // For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'c0'). + long n = Math.min(nCandidate, m); + sb.append(n); + sb.append(")"); + } + } } public static class StoneDBTable diff --git a/src/sqlancer/stonedb/StoneDBToStringVisitor.java b/src/sqlancer/stonedb/StoneDBToStringVisitor.java index 9d7711873..45b050b50 100644 --- a/src/sqlancer/stonedb/StoneDBToStringVisitor.java +++ b/src/sqlancer/stonedb/StoneDBToStringVisitor.java @@ -2,10 +2,12 @@ import sqlancer.common.ast.newast.NewToStringVisitor; import sqlancer.common.ast.newast.Node; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; import sqlancer.stonedb.ast.StoneDBJoin; import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; public class StoneDBToStringVisitor extends NewToStringVisitor { @Override @@ -16,6 +18,8 @@ public void visitSpecific(Node expr) { visit((StoneDBSelect) expr); } else if (expr instanceof StoneDBJoin) { visit((StoneDBJoin) expr); + } else if (expr instanceof StoneDBCastOperation) { + visit((StoneDBCastOperation) expr); } else { throw new AssertionError(expr.getClass()); } @@ -81,6 +85,14 @@ private void visit(StoneDBJoin join) { } } + private void visit(StoneDBCastOperation cast) { + sb.append("CAST("); + visit(cast.getExpr()); + sb.append(" AS "); + sb.append(cast.getType() == StoneDBDataType.INT ? "UNSIGNED" : cast.getType().toString()); + sb.append(") "); + } + public static String asString(Node expr) { StoneDBToStringVisitor visitor = new StoneDBToStringVisitor(); visitor.visit(expr); diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 385154815..87876c5cc 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -2,11 +2,14 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; +import java.util.Set; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.NewCaseOperatorNode; @@ -32,25 +35,35 @@ public StoneDBExpressionGenerator(StoneDBGlobalState globalState) { private enum Expression { UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, BINARY_BITWISE, BETWEEN, IN, - CASE + NOT_IN, CASE } - public static class StoneDBCastOperation extends NewUnaryPostfixOperatorNode { + public static class StoneDBCastOperation implements Node { + Node expr; + StoneDBDataType type; public StoneDBCastOperation(Node expr, StoneDBDataType type) { - super(expr, () -> "::" + type.toString()); + this.expr = expr; + this.type = type; } + public Node getExpr() { + return expr; + } + + public StoneDBDataType getType() { + return type; + } } @Override public Node negatePredicate(Node predicate) { - return null; + return new NewUnaryPrefixOperatorNode<>(predicate, StoneDBUnaryPrefixOperator.NOT); } @Override public Node isNull(Node expr) { - return null; + return new NewUnaryPostfixOperatorNode<>(expr, StoneDBUnaryPostfixOperator.IS_NULL); } @Override @@ -111,6 +124,12 @@ protected Node generateExpression(int depth) { case BINARY_COMPARISON: op = StoneDBBinaryComparisonOperator.getRandom(); return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); + case IN: + return new NewInOperatorNode<>(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), false); + case NOT_IN: + return new NewInOperatorNode<>(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), true); case BINARY_LOGICAL: op = StoneDBBinaryLogicalOperator.getRandom(); return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); @@ -123,9 +142,6 @@ protected Node generateExpression(int depth) { case BETWEEN: return new NewBetweenOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); - case IN: - return new NewInOperatorNode<>(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; return new NewCaseOperatorNode<>(generateExpression(depth + 1), generateExpressions(nr, depth + 1), @@ -137,7 +153,17 @@ protected Node generateExpression(int depth) { @Override protected Node generateColumn() { - return null; + StoneDBColumn column = Randomly.fromList(columns); + return new ColumnReferenceNode<>(column); + } + + protected List> generateColumns() { + int size = globalState.getRandomly().getInteger(1, columns.size()); + Set> set = new HashSet<>(); + while (set.size() < size) { + set.add(generateColumn()); + } + return new ArrayList<>(set); } public enum StoneDBAggregateFunction { @@ -160,7 +186,7 @@ public int getNrArgs() { public enum StoneDBUnaryPrefixOperator implements Operator { - NOT("NOT"), PLUS("+"), MINUS("-"); + NOT("NOT"), PLUS("+"), MINUS("-"), INVERSION("!"); private final String textRepr; @@ -203,8 +229,7 @@ public static StoneDBUnaryPostfixOperator getRandom() { */ public enum StoneDBBinaryComparisonOperator implements Operator { EQUAL("="), GREATER(">"), LESS("<"), GREATER_EQUAL(">="), LESS_EQUAL("<="), - NOT_EQUALS(Randomly.fromList(Arrays.asList("!=", "<>"))), NULL_SAFE_EQUAL("<=>"), IN("IN"), NOT_IN("NOT_IN"), - LIKE("LIKE"), IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + NOT_EQUALS(Randomly.fromList(Arrays.asList("!=", "<>"))), NULL_SAFE_EQUAL("<=>"), LIKE("LIKE"); private final String textRepr; @@ -227,7 +252,7 @@ public String getTextRepresentation() { */ public enum StoneDBBinaryLogicalOperator implements Operator { - NOT("NOT"), AND("AND"), OR("OR"), XOR("XOR"); + AND("AND"), OR("OR"), XOR("XOR"); private final String textRepr; @@ -274,7 +299,7 @@ public String getTextRepresentation() { * Bitwise operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/bitwise-operators */ public enum StoneDBBinaryBitwiseOperator implements Operator { - AND("&"), OR("|"), XOR("^"), INVERSION("!"), LEFTSHIFT("<<"), RIGHTSHIFT(">>"); + AND("&"), OR("|"), XOR("^"), LEFTSHIFT("<<"), RIGHTSHIFT(">>"); private final String textRepr; diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java index 3b7e63422..b50f07a6a 100644 --- a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java @@ -5,6 +5,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; public class StoneDBIndexCreateGenerator { @@ -30,10 +31,12 @@ private SQLQueryAdapter getQuery() { sb.append(Randomly.fromOptions("UNIQUE", "FULLTEXT", "SPATIAL")); sb.append(" INDEX "); sb.append(globalState.getSchema().getFreeIndexName()); - appendIndexType(); + if (Randomly.getBoolean()) { + appendIndexType(); + } sb.append(" ON "); sb.append(table.getName()); - appendKeyPart(); + appendKeyParts(); appendIndexOption(); appendAlgoOrLockOption(); addExpectedErrors(); @@ -67,9 +70,20 @@ private void appendIndexType() { sb.append(Randomly.fromOptions("BTREE", "HASH")); } + private void appendKeyParts() { + int numberOfKeyParts = Randomly.fromOptions(1, 1, 1, 1, table.getColumns().size()); + for (int i = 0; i < numberOfKeyParts; i++) { + appendKeyPart(); + } + } + private void appendKeyPart() { sb.append("("); - sb.append(table.getRandomColumn().getName()); + StoneDBColumn randomColumn = table.getRandomColumn(); + sb.append(randomColumn.getName()); + if (Randomly.getBoolean()) { + sb.append(" (").append(Randomly.smallNumber()).append(")"); + } if (Randomly.getBoolean()) { sb.append(Randomly.fromOptions(" ASC", " DESC")); } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index f25174948..b1e4872aa 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -1,24 +1,29 @@ package sqlancer.stonedb.gen; +import java.util.List; +import java.util.regex.Pattern; + import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBTable; public class StoneDBTableAlterGenerator { private final StoneDBGlobalState globalState; private final StringBuilder sb = new StringBuilder(); + private final StoneDBTable table; ExpectedErrors errors = new ExpectedErrors(); enum Action { - ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN, CHANGE_COLUMN, RENAME_COLUMN + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN, CHANGE_COLUMN } public StoneDBTableAlterGenerator(StoneDBGlobalState globalState) { this.globalState = globalState; + table = globalState.getSchema().getRandomTable(t -> !t.isView()); } public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { @@ -27,19 +32,56 @@ public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { private SQLQueryAdapter getQuery() { sb.append("ALTER TABLE "); - StoneDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) - .setColumns(table.getColumns()); sb.append(table.getName()); sb.append(" "); - Action action = Randomly.fromOptions(Action.values()); + appendAlterOptions(); + addExpectedErrors(); + return new SQLQueryAdapter(sb.toString(), errors, true); + } + + private void addExpectedErrors() { + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Data too long for column 'c0' at row 2 + errors.addRegex(Pattern.compile("Data truncation: Data too long for column 'c\\d{1,3}' at row \\d{1,3}")); + // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes + errors.add("Specified key was too long; max key length is 3072 bytes"); + // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead + errors.add("You can't delete all columns with ALTER TABLE; use DROP TABLE instead"); + // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 't1' + errors.addRegex(Pattern.compile("Unknown column 'c\\d{1,3}' in 't\\d{1,3}'")); + // java.sql.SQLSyntaxErrorException: BLOB, TEXT, GEOMETRY or JSON column 'c0' can't have a default value + errors.addRegex(Pattern.compile("BLOB, TEXT, GEOMETRY or JSON column 'c\\d{1,3}' can't have a default value")); + // java.sql.SQLSyntaxErrorException: Column length too big for column 'c91' (max = 16383); use BLOB or TEXT + // instead + errors.addRegex(Pattern + .compile("Column length too big for column 'c\\d{1,3}' \\(max = 16383\\); use BLOB or TEXT instead")); + } + + private void appendAlterOptions() { + List actions; + if (Randomly.getBooleanWithSmallProbability()) { + actions = Randomly.subset(Action.values()); + } else { + actions = List.of(Randomly.fromOptions(Action.values())); + } + for (Action action : actions) { + appendAlterOption(action); + sb.append(" "); + } + } + + private void appendAlterOption(Action action) { + StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) + .setColumns(table.getColumns()); switch (action) { case ADD_COLUMN: sb.append("ADD COLUMN "); String columnName = table.getFreeColumnName(); - sb.append(columnName); - sb.append(" "); - sb.append(StoneDBCompositeDataType.getRandomWithoutNull().getPrimitiveDataType().toString()); + sb.append(" ").append(columnName).append(" "); + sb.append(StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull())); + // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT + // instead + errors.addRegex(Pattern + .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); @@ -63,17 +105,17 @@ private SQLQueryAdapter getQuery() { } else { sb.append(" DROP DEFAULT"); } - if (Randomly.getBoolean()) { - sb.append(" SET ").append(Randomly.fromOptions("VISIBLE", "INVISIBLE")); - } break; case CHANGE_COLUMN: sb.append(Randomly.fromOptions("CHANGE COLUMN ", "CHANGE ")); String oldColumnName = table.getRandomColumn().getName(); String newColumnName = table.getFreeColumnName(); - sb.append(oldColumnName).append(" ").append(newColumnName); - sb.append(" "); - sb.append(StoneDBCompositeDataType.getRandomWithoutNull().getPrimitiveDataType().toString()); + sb.append(oldColumnName).append(" ").append(newColumnName).append(" "); + sb.append(StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull())); + // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT + // instead + errors.addRegex(Pattern + .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); @@ -83,21 +125,8 @@ private SQLQueryAdapter getQuery() { } } break; - case RENAME_COLUMN: - sb.append("RENAME COLUMN "); - sb.append(table.getRandomColumn().getName()); - sb.append(" TO "); - sb.append(table.getFreeColumnName()); - break; default: throw new AssertionError(action); } - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void addExpectedErrors() { - // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead - errors.add("You can't delete all columns with ALTER TABLE; use DROP TABLE instead"); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java index 622c86c3c..3b87e99d5 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java @@ -1,12 +1,13 @@ package sqlancer.stonedb.gen; import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.PRIMARY_KEY; -import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.UNIQUE; +import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.UNIQUE_KEY; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.regex.Pattern; import sqlancer.Randomly; import sqlancer.Randomly.StringGenerationStrategy; @@ -41,7 +42,7 @@ public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String ta } public SQLQueryAdapter getQuery() { - sb.append(Randomly.fromOptions("CREATE TABLE ", "CREATE TEMPORARY TABLE ")); + sb.append(Randomly.fromOptions("CREATE TABLE "/* , "CREATE TEMPORARY TABLE " */)); if (Randomly.getBoolean()) { sb.append("IF NOT EXISTS "); } @@ -52,8 +53,10 @@ public SQLQueryAdapter getQuery() { sb.append(schema.getRandomTable().getName()); } else { appendColumns(); - sb.append(" "); - appendTableOptions(); + if (Randomly.getBoolean()) { + sb.append(" "); + appendTableOptions(); + } } addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors, true); @@ -71,6 +74,8 @@ private void addExpectedErrors() { // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT // instead errors.add("Column length too big for column"); + // BLOB/TEXT column 'c0' used in key specification without a key length + errors.addRegex(Pattern.compile("BLOB/TEXT column 'c.*' used in key specification without a key length")); } private enum TableOptions { @@ -187,7 +192,7 @@ private void appendColumnDefinition() { } protected enum ColumnOptions { - NULL_OR_NOT_NULL, UNIQUE, COMMENT, COLUMN_FORMAT, STORAGE, PRIMARY_KEY + NULL_OR_NOT_NULL, PRIMARY_KEY, UNIQUE_KEY, COMMENT, COLUMN_FORMAT, STORAGE } private void appendColumnOption(StoneDBDataType type) { @@ -200,13 +205,13 @@ private void appendColumnOption(StoneDBDataType type) { // tableHasNullableColumn = true; // } // only use one key, unique key or primary key, but not both - if (columnOptions.contains(PRIMARY_KEY) && columnOptions.contains(UNIQUE)) { - columnOptions.remove(Randomly.fromOptions(PRIMARY_KEY, UNIQUE)); + if (columnOptions.contains(PRIMARY_KEY) && columnOptions.contains(UNIQUE_KEY)) { + columnOptions.remove(Randomly.fromOptions(PRIMARY_KEY, UNIQUE_KEY)); } if (isTextType) { // TODO: restriction due to the limited key length columnOptions.remove(PRIMARY_KEY); - columnOptions.remove(UNIQUE); + columnOptions.remove(UNIQUE_KEY); } for (ColumnOptions o : columnOptions) { sb.append(" "); @@ -223,7 +228,7 @@ private void appendColumnOption(StoneDBDataType type) { sb.append("NOT NULL"); } break; - case UNIQUE: + case UNIQUE_KEY: sb.append("UNIQUE"); if (Randomly.getBoolean()) { sb.append(" KEY"); @@ -256,108 +261,6 @@ private void appendColumnOption(StoneDBDataType type) { } private void appendType(StoneDBDataType randomType) { - switch (randomType) { - case TINYINT: - sb.append("TINYINT"); - // sb.append(r.getInteger(-128, 127)); - break; - case SMALLINT: - sb.append("SMALLINT"); - // sb.append(r.getInteger(-32768, 32767)); - break; - case MEDIUMINT: - sb.append("MEDIUMINT"); - // sb.append(r.getInteger(-8388608, 8388607)); - break; - case INT: - sb.append("INT"); - // sb.append(r.getInteger(-2147483647, 2147483647)); - break; - case BIGINT: - sb.append("BIGINT"); - // sb.append(r.getBigInteger(new BigInteger("-9223372036854775806"), new - // BigInteger("9223372036854775807"))); - break; - case FLOAT: - sb.append("FLOAT"); - optionallyAddPrecisionAndScale(sb); - break; - case DOUBLE: - sb.append("DOUBLE"); - optionallyAddPrecisionAndScale(sb); - break; - case DECIMAL: - sb.append("DECIMAL"); // The default value is P(10,0); - break; - case YEAR: - sb.append("YEAR"); - break; - case TIME: - sb.append("TIME"); - break; - case DATE: - sb.append("DATE"); - break; - case DATETIME: - sb.append("DATETIME"); - break; - case TIMESTAMP: - sb.append("TIMESTAMP"); - break; - case CHAR: - sb.append("CHAR").append(Randomly.fromOptions("", "(" + r.getInteger(0, 255) + ")")); - break; - case VARCHAR: - sb.append("VARCHAR").append("(").append(r.getInteger(0, 65535)).append(")"); - break; - case TINYTEXT: - sb.append("TINYTEXT"); - break; - case TEXT: - sb.append("TEXT"); - break; - case MEDIUMTEXT: - sb.append("MEDIUMTEXT"); - break; - case LONGTEXT: - sb.append("LONGTEXT"); - break; - case BINARY: - sb.append("BINARY"); - break; - case VARBINARY: - sb.append("VARBINARY").append("(").append(r.getInteger(0, 65535)).append(")"); - break; - case TINYBLOB: - sb.append("TINYBLOB"); - break; - case BLOB: - sb.append("BLOB"); - break; - case MEDIUMBLOB: - sb.append("MEDIUMBLOB"); - break; - case LONGBLOB: - sb.append("LONGBLOB"); - break; - default: - throw new AssertionError(); - } - } - - public static void optionallyAddPrecisionAndScale(StringBuilder sb) { - if (Randomly.getBoolean()) { - sb.append("("); - // The maximum number of digits (M) for DECIMAL is 65 - long m = Randomly.getNotCachedInteger(1, 65); - sb.append(m); - sb.append(", "); - // The maximum number of supported decimals (D) is 30 - long nCandidate = Randomly.getNotCachedInteger(1, 30); - // For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'c0'). - long n = Math.min(nCandidate, m); - sb.append(n); - sb.append(")"); - } + sb.append(StoneDBDataType.getTypeAndValue(randomType)); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index 3ff809740..ab0828d30 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -45,7 +45,7 @@ public SQLQueryAdapter getQuery() { sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) .setColumns(randomTable.getColumns()).generateExpression())); } - if (Randomly.getBoolean()) { + if (!StoneDBBugs.bug1933 && Randomly.getBoolean()) { sb.append(" ORDER BY "); sb.append(String.join(", ", Randomly.fromOptions( randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))) @@ -55,13 +55,6 @@ public SQLQueryAdapter getQuery() { sb.append(" LIMIT "); sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); } - addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } - - private void addExpectedErrors() { - if (StoneDBBugs.bug1933) { - errors.add("assert failed on i < m_idx.size() at tianmu_attr.h:387, msg: [bad dpn index 0/0]"); - } - } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index 7a0e3847f..2fdb7b6fa 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -1,6 +1,7 @@ package sqlancer.stonedb.gen; import java.util.List; +import java.util.regex.Pattern; import java.util.stream.Collectors; import sqlancer.Randomly; @@ -57,6 +58,10 @@ private void addExpectedErrors() { errors.add("Insert duplicate key on row: "); // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: errors.add("Data truncation: Incorrect datetime value: "); + // java.sql.SQLException: Field 'c0' doesn't have a default value + errors.add("doesn't have a default value"); + // java.sql.SQLException: Data truncated for column 'c0' at row 1 + errors.addRegex(Pattern.compile("Data truncated for column 'c.*' at row .*")); } private void appendPartition() { diff --git a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java index bff098a64..4812a6ee3 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java @@ -49,8 +49,11 @@ public void appendAssignmentList() { } sb.append(table.getColumns().get(i)); sb.append(" = "); - sb.append(StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType - .getRandomValue(table.getColumns().get(i).getType().getPrimitiveDataType()))); + sb.append( + Randomly.fromOptions( + StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType + .getRandomValue(table.getColumns().get(i).getType().getPrimitiveDataType())), + "DEFAULT")); } } diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 1d3dac312..2625bbb7c 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -11,6 +11,7 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.NewPostfixTextNode; import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; @@ -29,6 +30,7 @@ import sqlancer.stonedb.ast.StoneDBJoin; import sqlancer.stonedb.ast.StoneDBSelect; import sqlancer.stonedb.gen.StoneDBExpressionGenerator; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBBinaryLogicalOperator; import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; public class StoneDBNoRECOracle extends NoRECBase implements TestOracle { @@ -65,14 +67,16 @@ public void check() throws Exception { private int getUnoptimizedQueryCount(List> tableList, Node randomWhereCondition, List> joins) throws SQLException { StoneDBSelect select = new StoneDBSelect(); - Node asText = new NewPostfixTextNode<>(new StoneDBCastOperation( - new NewPostfixTextNode<>(randomWhereCondition, - " IS NOT NULL AND " + StoneDBToStringVisitor.asString(randomWhereCondition)), - StoneDBDataType.INT), "as count"); + Node asText = new NewPostfixTextNode<>( + new StoneDBCastOperation( + new NewBinaryOperatorNode<>(new NewPostfixTextNode<>(randomWhereCondition, " IS NOT NULL "), + randomWhereCondition, StoneDBBinaryLogicalOperator.AND), + StoneDBDataType.INT), + " as count"); select.setFetchColumns(List.of(asText)); select.setFromList(tableList); select.setJoinList(joins); - unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res"; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { @@ -104,14 +108,16 @@ private int getOptimizedQueryCount(SQLConnection con, List Date: Sun, 23 Jul 2023 23:15:40 +0800 Subject: [PATCH 373/856] feat: add TLP support of StoneDB (#857) * fix: Tianmu engine does not support fulltext index. * feat: add expectedErrors for all generator * refactor: remove unused expected errors * feat: add expectedErrors for all generator * refactor: remove unused expected errors * feat: init support of TLP for StoneDB * fix: mvn test errors * feat: add CI * feat: add CI * feat: add CI * fix: remove unsupported operator * fix: change string generator strategy * feat: add expected errors * fix: binary type should have a max length * fix: mvn format * fix: MySQL 5.7 does not support double, change to decimal * fix: natural join type * fix: only one alter option one time * feat: add expected errors * git: manually revert commits * git: manually revert commits * fix: use latest version and set tianmu_insert_delayed=0 * rename: rename oracle --- .github/workflows/main.yml | 41 ++++- src/sqlancer/stonedb/StoneDBErrors.java | 1 - src/sqlancer/stonedb/StoneDBOptions.java | 55 +++++- .../gen/StoneDBExpressionGenerator.java | 11 ++ .../oracle/StoneDBAggregateOracle.java | 168 ++++++++++++++++++ .../oracle/StoneDBQueryPartitioningBase.java | 71 ++++++++ ...toneDBQueryPartitioningDistinctTester.java | 42 +++++ ...StoneDBQueryPartitioningGroupByTester.java | 50 ++++++ .../StoneDBQueryPartitioningHavingTester.java | 46 +++++ .../StoneDBQueryPartitioningWhereTester.java | 42 +++++ .../stonedb/oracle/StoneDBTLPOracle.java | 16 -- ...TestStoneDB.java => TestStoneDBNoRec.java} | 2 +- test/sqlancer/dbms/TestStoneDBTLP.java | 21 +++ 13 files changed, 534 insertions(+), 32 deletions(-) create mode 100644 src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java create mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java rename test/sqlancer/dbms/{TestStoneDB.java => TestStoneDBNoRec.java} (95%) create mode 100644 test/sqlancer/dbms/TestStoneDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 990543c0d..9aaed3c70 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -375,8 +375,8 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test - stonedb: - name: DBMS Tests (StoneDB) + stonedb-norec: + name: DBMS Tests (StoneDB NoRec) runs-on: ubuntu-latest steps: @@ -385,12 +385,11 @@ jobs: fetch-depth: 0 - name: Set up StoneDB run: | - docker pull stoneatom/stonedb:v1.0.3 - docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb:v1.0.3 - docker ps - ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb:v1.0.3 --format "{{.ID}}") + docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb + ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb --format "{{.ID}}") docker exec $ContainerID bash - sleep 180s + sleep 60s + docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Set up JDK 11 uses: actions/setup-java@v1.4.4 @@ -400,7 +399,33 @@ jobs: run: mvn -B package -DskipTests=true - name: StoneDB Tests run: | - STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDB + STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBNoRec + + stonedb-tlp: + name: DBMS Tests (StoneDB TLP) + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up StoneDB + run: | + docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb + ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb --format "{{.ID}}") + docker exec $ContainerID bash + sleep 60s + docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf + docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + - name: Set up JDK 11 + uses: actions/setup-java@v1.4.4 + with: + java-version: 11 + - name: Build + run: mvn -B package -DskipTests=true + - name: StoneDB Tests + run: | + STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBTLP tidb: name: DBMS Tests (TiDB) diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java index 54fd28ad8..bdbd6a447 100644 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -3,5 +3,4 @@ public final class StoneDBErrors { private StoneDBErrors() { } - } diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java index 928bcfa39..7e6a2f09b 100644 --- a/src/sqlancer/stonedb/StoneDBOptions.java +++ b/src/sqlancer/stonedb/StoneDBOptions.java @@ -1,7 +1,7 @@ package sqlancer.stonedb; import java.sql.SQLException; -import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import com.beust.jcommander.Parameter; @@ -9,11 +9,16 @@ import sqlancer.DBMSSpecificOptions; import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.stonedb.StoneDBOptions.StoneDBOracleFactory; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.oracle.StoneDBAggregateOracle; import sqlancer.stonedb.oracle.StoneDBNoRECOracle; -import sqlancer.stonedb.oracle.StoneDBTLPOracle; +import sqlancer.stonedb.oracle.StoneDBQueryPartitioningDistinctTester; +import sqlancer.stonedb.oracle.StoneDBQueryPartitioningGroupByTester; +import sqlancer.stonedb.oracle.StoneDBQueryPartitioningHavingTester; +import sqlancer.stonedb.oracle.StoneDBQueryPartitioningWhereTester; @Parameters(separators = "=", commandDescription = "StoneDB (default host: " + StoneDBOptions.DEFAULT_HOST + ", default port: " + StoneDBOptions.DEFAULT_PORT + ")") @@ -22,7 +27,7 @@ public class StoneDBOptions implements DBMSSpecificOptions public static final int DEFAULT_PORT = 3306; @Parameter(names = "--oracle") - public List oracles = Arrays.asList(StoneDBOracleFactory.NOREC); + public List oracles = List.of(StoneDBOracleFactory.NOREC); public enum StoneDBOracleFactory implements OracleFactory { NOREC { @@ -31,10 +36,48 @@ public TestOracle create(StoneDBGlobalState globalState) thr return new StoneDBNoRECOracle(globalState); } }, - TLP { + QUERY_PARTITIONING { @Override - public TestOracle create(StoneDBGlobalState globalState) throws Exception { - return new StoneDBTLPOracle(globalState); + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); + oracles.add(new StoneDBQueryPartitioningWhereTester(globalState)); + oracles.add(new StoneDBQueryPartitioningHavingTester(globalState)); + oracles.add(new StoneDBAggregateOracle(globalState)); + oracles.add(new StoneDBQueryPartitioningDistinctTester(globalState)); + oracles.add(new StoneDBQueryPartitioningGroupByTester(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + }, + HAVING { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBAggregateOracle(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBQueryPartitioningDistinctTester(globalState); } } } diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 87876c5cc..54d2c4113 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -166,6 +166,7 @@ protected List> generateColumns() { return new ArrayList<>(set); } + // https://stonedb.io/docs/SQL-reference/functions/aggregate-functions/ public enum StoneDBAggregateFunction { MAX(1), MIN(1), AVG(1), COUNT(1), FIRST(1), SUM(1); @@ -317,4 +318,14 @@ public String getTextRepresentation() { } } + + public NewFunctionNode generateAggregateAndArgs( + StoneDBAggregateFunction aggregateFunction) { + return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + } + + public Node generateAggregate() { + StoneDBAggregateFunction aggrFunc = StoneDBAggregateFunction.getRandom(); + return generateAggregateAndArgs(aggrFunc); + } } diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java new file mode 100644 index 000000000..df09e8961 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -0,0 +1,168 @@ +package sqlancer.stonedb.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; +import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBAggregateFunction; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPostfixOperator; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPrefixOperator; + +public class StoneDBAggregateOracle extends StoneDBQueryPartitioningBase { + + public StoneDBAggregateOracle(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + + StoneDBAggregateFunction aggregateFunction = Randomly.fromOptions(StoneDBAggregateFunction.values()); + NewFunctionNode aggregate = gen + .generateAggregateAndArgs(aggregateFunction); + + List> fetchColumns = new ArrayList<>(); + fetchColumns.add(aggregate); + select.setFetchColumns(fetchColumns); + + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + + String originalQuery = StoneDBToStringVisitor.asString(select); + String originalResult = getAggregateResult(originalQuery); + + String metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + String metamorphicResult = getAggregateResult(metamorphicQuery); + + String line1 = "--" + originalQuery + ";"; + String line2 = "--" + originalResult + ";"; + String line3 = "--" + originalResult + ";"; + String line4 = "--" + metamorphicResult + ";"; + String output = String.join(System.lineSeparator(), line1, line2, line3, line4); + state.getState().getLocalState().log(output); + + if (originalResult == null && metamorphicResult != null + || originalResult != null && (!originalResult.contentEquals(metamorphicResult) + && !ComparatorHelper.isEqualDouble(originalResult, metamorphicResult))) { + throw new AssertionError("aggregate result mismatch!" + System.lineSeparator() + output); + } + } + + private String createMetamorphicUnionQuery(StoneDBSelect select, + NewFunctionNode aggregate, + List> from) { + String metamorphicQuery; + Node whereClause = gen.generateExpression(); + + Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + StoneDBUnaryPrefixOperator.NOT); + Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + StoneDBUnaryPostfixOperator.IS_NULL); + List> mappedAggregate = mapped(aggregate); + StoneDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + StoneDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + StoneDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += StoneDBToStringVisitor.asString(leftSelect) + " UNION ALL " + + StoneDBToStringVisitor.asString(middleSelect) + " UNION ALL " + + StoneDBToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as result"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) throws SQLException { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List> mapped( + NewFunctionNode aggregate) { + + StoneDBCastOperation count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(List.of(aggregate)); + case AVG: + NewFunctionNode sum = new NewFunctionNode<>( + aggregate.getArgs(), StoneDBAggregateFunction.SUM); + count = new StoneDBCastOperation(new NewFunctionNode<>(aggregate.getArgs(), StoneDBAggregateFunction.COUNT), + new StoneDBCompositeDataType(StoneDBDataType.DECIMAL).getPrimitiveDataType()); + return aliasArgs(Arrays.asList(sum, count)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List> aliasArgs(List> originalAggregateArgs) { + List> args = new ArrayList<>(); + int i = 0; + for (Node expr : originalAggregateArgs) { + args.add(new NewAliasNode(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(NewFunctionNode aggregate) { + switch (aggregate.getFunc()) { + case AVG: + return "SUM(agg0)/SUM(agg1)"; + case COUNT: + return StoneDBAggregateFunction.SUM + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private StoneDBSelect getSelect(List> aggregates, List> fromList, + Node whereClause, List> joinList) { + StoneDBSelect select = new StoneDBSelect(); + select.setFetchColumns(aggregates); + select.setFromList(fromList); + select.setWhereClause(whereClause); + select.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return select; + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java new file mode 100644 index 000000000..84d9f794f --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java @@ -0,0 +1,71 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBSchema.StoneDBTables; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator; + +public class StoneDBQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, StoneDBGlobalState> + implements TestOracle { + + StoneDBSchema schema; + StoneDBTables targetTables; + StoneDBExpressionGenerator gen; + StoneDBSelect select; + + public StoneDBQueryPartitioningBase(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + schema = state.getSchema(); + targetTables = schema.getRandomTableNonEmptyTables(); + gen = new StoneDBExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + + select = new StoneDBSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableReferenceNodeList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = StoneDBJoin.getJoins(tableReferenceNodeList, state); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableReferenceNodeList)); + select.setWhereClause(null); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..cba8290b4 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java @@ -0,0 +1,42 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBQueryPartitioningDistinctTester extends StoneDBQueryPartitioningBase { + public StoneDBQueryPartitioningDistinctTester(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = StoneDBToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + if (Randomly.getBoolean()) { + select.setDistinct(false); + } + + select.setWhereClause(predicate); + String firstQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = StoneDBToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..d7674a71c --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java @@ -0,0 +1,50 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; + +public class StoneDBQueryPartitioningGroupByTester extends StoneDBQueryPartitioningBase { + public StoneDBQueryPartitioningGroupByTester(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + // common to both original and combined query string + select.setGroupByExpressions(select.getFetchColumns()); + // specific to original query string + select.setWhereClause(null); + String originalQueryString = StoneDBToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + // specific to combined query string, will produce the same result as original query string in logic + select.setWhereClause(predicate); + String firstQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = StoneDBToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + // compare the result + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } + + @Override + List> generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java new file mode 100644 index 000000000..ea611241b --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java @@ -0,0 +1,46 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBQueryPartitioningHavingTester extends StoneDBQueryPartitioningBase { + public StoneDBQueryPartitioningHavingTester(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + // common to both original and combined query string + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + // specific to original query string + select.setHavingClause(null); + String originalQueryString = StoneDBToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + // specific to combined query string, will produce the same result as original query string in logic + select.setHavingClause(predicate); + String firstQueryString = StoneDBToStringVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = StoneDBToStringVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = StoneDBToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + // compare the result + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java new file mode 100644 index 000000000..2a8f2b4d4 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java @@ -0,0 +1,42 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBToStringVisitor; + +public class StoneDBQueryPartitioningWhereTester extends StoneDBQueryPartitioningBase { + public StoneDBQueryPartitioningWhereTester(StoneDBGlobalState state) { + super(state); + } + + @Override + public void check() throws Exception { + super.check(); + + select.setWhereClause(null); + String originalQueryString = StoneDBToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setWhereClause(predicate); + String firstQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = StoneDBToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = StoneDBToStringVisitor.asString(select); + + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + // compare the result + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } +} diff --git a/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java b/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java deleted file mode 100644 index 96deda89e..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBTLPOracle.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.stonedb.oracle; - -import sqlancer.common.oracle.NoRECBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; - -public class StoneDBTLPOracle extends NoRECBase implements TestOracle { - public StoneDBTLPOracle(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - - } -} diff --git a/test/sqlancer/dbms/TestStoneDB.java b/test/sqlancer/dbms/TestStoneDBNoRec.java similarity index 95% rename from test/sqlancer/dbms/TestStoneDB.java rename to test/sqlancer/dbms/TestStoneDBNoRec.java index 14de121be..4b158f372 100644 --- a/test/sqlancer/dbms/TestStoneDB.java +++ b/test/sqlancer/dbms/TestStoneDBNoRec.java @@ -7,7 +7,7 @@ import sqlancer.Main; -public class TestStoneDB { +public class TestStoneDBNoRec { String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); diff --git a/test/sqlancer/dbms/TestStoneDBTLP.java b/test/sqlancer/dbms/TestStoneDBTLP.java new file mode 100644 index 000000000..d5db5b9ab --- /dev/null +++ b/test/sqlancer/dbms/TestStoneDBTLP.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestStoneDBTLP { + + String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); + boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); + + @Test + public void testStoneDB() { + assumeTrue(stoneDBIsAvailable); + assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", + "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "QUERY_PARTITIONING")); + } + +} From 742ce6f48460f88a9dc9dafbf6213ddfbefcadc6 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:21:11 +0800 Subject: [PATCH 374/856] feat: add bugs found (#868) --- src/sqlancer/stonedb/StoneDBBugs.java | 36 +++++++++++++++++++ src/sqlancer/stonedb/StoneDBProvider.java | 5 +++ src/sqlancer/stonedb/ast/StoneDBConstant.java | 5 +++ .../gen/StoneDBExpressionGenerator.java | 18 ++++++++-- .../gen/StoneDBTableInsertGenerator.java | 7 ++-- .../stonedb/oracle/StoneDBNoRECOracle.java | 5 +++ 6 files changed, 72 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java index 01040c429..11a144dad 100644 --- a/src/sqlancer/stonedb/StoneDBBugs.java +++ b/src/sqlancer/stonedb/StoneDBBugs.java @@ -3,6 +3,42 @@ public final class StoneDBBugs { // https://github.com/stoneatom/stonedb/issues/1933 public static boolean bug1933 = true; + // https://github.com/stoneatom/stonedb/issues/1942 + public static boolean bug1942 = true; + // https://github.com/stoneatom/stonedb/issues/1953 + public static boolean bug1953 = true; + // https://github.com/stoneatom/stonedb/issues/1945 + public static boolean bug1945 = true; + // CREATE TABLE t0(c0 INT); + // INSERT IGNORE INTO t0(c0) VALUE (DEFAULT); + // SELECT t0.c0 FROM t0 WHERE 0.4; -- expect 1 but got 0 + public static boolean bugNotReported1 = true; + // DELETE statements will result into crash, for example + // DELETE LOW_PRIORITY FROM t0; + // DELETE QUICK IGNORE FROM t0 WHERE -1370759901; + public static boolean bugNotReported2 = true; + // CREATE TABLE t0(c0 INT) ; + // INSERT INTO t0(c0) VALUES (DEFAULT); + // SELECT t0.c0 FROM t0 WHERE (('OC')>=(((t0.c0) IS NULL))); -- expected empty set but got 1 row + public static boolean bugNotReported3 = true; + // CREATE TABLE t0(c0 CHAR PRIMARY KEY); + // INSERT IGNORE INTO t0(c0) VALUE (DEFAULT); + // SELECT t0.c0 FROM t0 WHERE (t0.c0 BETWEEN (0) AND t0.c0); -- 1 + public static boolean bugNotReported4 = true; + // CREATE TABLE t0(c0 INT); + // INSERT INTO t0(c0) VALUE (DEFAULT); + // SELECT * FROM t0 WHERE (('00')>((('')AND(t0.c0)))); -- expected empty set but got 1 row + public static boolean bugNotReported5 = true; + // CREATE TABLE t0(c0 INT) ; + // INSERT INTO t0(c0) VALUE (100); + // SELECT * FROM t0 WHERE (('Xg')>(t0.c0)); -- expected empty set but got 1 row + // SELECT (('Xg')>(t0.c0)) FROM t0; + public static boolean bugNotReported6 = true; + // CREATE TABLE t0(c0 VARCHAR(1000)); + // INSERT INTO t0(c0) VALUE ('GX9vartD'); + // INSERT INTO t0(c0) VALUE ('16'); + // SELECT t0.c0 FROM t0 WHERE t0.c0; -- expected 1 row but got 2 rows + public static boolean bugNotReported7 = true; private StoneDBBugs() { } diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java index e0c317c53..55b7aff70 100644 --- a/src/sqlancer/stonedb/StoneDBProvider.java +++ b/src/sqlancer/stonedb/StoneDBProvider.java @@ -1,5 +1,7 @@ package sqlancer.stonedb; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported2; + import java.sql.Connection; import java.sql.DriverManager; import java.sql.Statement; @@ -76,6 +78,9 @@ private static int mapActions(StoneDBGlobalState globalState, Action a) { case TABLE_CREATE: return r.getInteger(0, 1); case TABLE_DELETE: + if (bugNotReported2) { + return 0; + } return r.getInteger(0, 10); case TABLE_INSERT: return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); diff --git a/src/sqlancer/stonedb/ast/StoneDBConstant.java b/src/sqlancer/stonedb/ast/StoneDBConstant.java index f0abae48a..6760bc25c 100644 --- a/src/sqlancer/stonedb/ast/StoneDBConstant.java +++ b/src/sqlancer/stonedb/ast/StoneDBConstant.java @@ -1,5 +1,7 @@ package sqlancer.stonedb.ast; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported1; + import java.sql.Timestamp; import java.text.SimpleDateFormat; @@ -85,6 +87,9 @@ public String toString() { } public static Node createDoubleConstant(double val) { + if (bugNotReported1 && val < 0.5 && val > 0) { + return new StoneDBDoubleConstant(0.5); + } return new StoneDBDoubleConstant(val); } diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 54d2c4113..6ed9d59bb 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -1,5 +1,9 @@ package sqlancer.stonedb.gen; +import static sqlancer.stonedb.StoneDBBugs.bug1942; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported3; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported6; + import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; @@ -119,10 +123,20 @@ protected Node generateExpression(int depth) { op = StoneDBUnaryPrefixOperator.getRandom(); return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), op); case UNARY_POSTFIX: - op = StoneDBUnaryPostfixOperator.getRandom(); + if (!bug1942) { + op = StoneDBUnaryPostfixOperator.getRandom(); + } else if (!bugNotReported3) { + op = StoneDBUnaryPostfixOperator.IS_NULL; + } else { + throw new IgnoreMeException(); + } return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), op); case BINARY_COMPARISON: - op = StoneDBBinaryComparisonOperator.getRandom(); + if (!bugNotReported6) { + op = StoneDBBinaryComparisonOperator.getRandom(); + } else { + throw new IgnoreMeException(); + } return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); case IN: return new NewInOperatorNode<>(generateExpression(depth + 1), diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index 2fdb7b6fa..f4435ec26 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -1,5 +1,8 @@ package sqlancer.stonedb.gen; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported4; +import static sqlancer.stonedb.StoneDBBugs.bugNotReported5; + import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -38,7 +41,7 @@ private SQLQueryAdapter getQuery() { sb.append(" "); sb.append(Randomly.fromOptions("LOW_PRIORITY", "DELAYED", "HIGH_PRIORITY")); } - if (Randomly.getBoolean()) { + if (!bugNotReported4 && Randomly.getBoolean()) { sb.append(" IGNORE"); } sb.append(" INTO "); @@ -107,7 +110,7 @@ private void appendOneValue(int nrRow) { // append one column of one row @Override protected void insertValue(StoneDBColumn column) { - if (Randomly.getBooleanWithRatherLowProbability()) { + if (!bugNotReported5 && Randomly.getBooleanWithRatherLowProbability()) { sb.append("DEFAULT"); } else { sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 2625bbb7c..534bf030d 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -1,5 +1,7 @@ package sqlancer.stonedb.oracle; +import static sqlancer.stonedb.StoneDBBugs.bug1953; + import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -77,6 +79,9 @@ private int getUnoptimizedQueryCount(List> tableList, select.setFromList(tableList); select.setJoinList(joins); unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; + if (bug1953) { + unoptimizedQueryString = "SELECT * FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; + } SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); SQLancerResultSet rs; try { From ea85c22fd8611deed2f5aee32d9499bb1a915811 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:21:40 +0800 Subject: [PATCH 375/856] fix: fix implementation bugs of randomly (#873) --- src/sqlancer/stonedb/StoneDBSchema.java | 12 ++++++------ .../stonedb/gen/StoneDBTableAlterGenerator.java | 6 ++++-- .../stonedb/gen/StoneDBTableCreateGenerator.java | 2 +- .../stonedb/gen/StoneDBTableUpdateGenerator.java | 9 ++++----- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java index 2150cbba1..346031fba 100644 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ b/src/sqlancer/stonedb/StoneDBSchema.java @@ -35,7 +35,7 @@ public static StoneDBDataType getRandom() { return Randomly.fromOptions(values()); } - public static Node getRandomValue(StoneDBDataType dataType) { + public static Node getRandomValue(StoneDBDataType dataType, Randomly r) { switch (dataType) { case TINYINT: break; @@ -70,7 +70,7 @@ public static Node getRandomValue(StoneDBDataType dataType) { case TINYTEXT: break; case TEXT: - return StoneDBConstant.createTextConstant(new Randomly().getString()); + return StoneDBConstant.createTextConstant(r.getString()); case MEDIUMTEXT: break; case LONGTEXT: @@ -93,7 +93,7 @@ public static Node getRandomValue(StoneDBDataType dataType) { return null; } - public static String getTypeAndValue(StoneDBDataType dataType) { + public static String getTypeAndValue(StoneDBDataType dataType, Randomly r) { StringBuilder sb = new StringBuilder(); switch (dataType) { case TINYINT: @@ -127,10 +127,10 @@ public static String getTypeAndValue(StoneDBDataType dataType) { case TIMESTAMP: return "TIMESTAMP"; case CHAR: - sb.append("CHAR").append(Randomly.fromOptions("", "(" + new Randomly().getInteger(0, 255) + ")")); + sb.append("CHAR").append(Randomly.fromOptions("", "(" + r.getInteger(0, 255) + ")")); return sb.toString(); case VARCHAR: - sb.append("VARCHAR").append("(").append(new Randomly().getInteger(0, 65535)).append(")"); + sb.append("VARCHAR").append("(").append(r.getInteger(0, 65535)).append(")"); return sb.toString(); case TINYTEXT: return "TINYTEXT"; @@ -143,7 +143,7 @@ public static String getTypeAndValue(StoneDBDataType dataType) { case BINARY: return "BINARY"; case VARBINARY: - sb.append("VARBINARY").append("(").append(new Randomly().getInteger(0, 65535)).append(")"); + sb.append("VARBINARY").append("(").append(r.getInteger(0, 65535)).append(")"); return sb.toString(); case TINYBLOB: return "TINYBLOB"; diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index b1e4872aa..0510f7a78 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -77,7 +77,8 @@ private void appendAlterOption(Action action) { sb.append("ADD COLUMN "); String columnName = table.getFreeColumnName(); sb.append(" ").append(columnName).append(" "); - sb.append(StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull())); + sb.append( + StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT // instead errors.addRegex(Pattern @@ -111,7 +112,8 @@ private void appendAlterOption(Action action) { String oldColumnName = table.getRandomColumn().getName(); String newColumnName = table.getFreeColumnName(); sb.append(oldColumnName).append(" ").append(newColumnName).append(" "); - sb.append(StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull())); + sb.append( + StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT // instead errors.addRegex(Pattern diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java index 3b87e99d5..c288c57fd 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java @@ -261,6 +261,6 @@ private void appendColumnOption(StoneDBDataType type) { } private void appendType(StoneDBDataType randomType) { - sb.append(StoneDBDataType.getTypeAndValue(randomType)); + sb.append(StoneDBDataType.getTypeAndValue(randomType, r)); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java index 4812a6ee3..259136135 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java @@ -49,11 +49,10 @@ public void appendAssignmentList() { } sb.append(table.getColumns().get(i)); sb.append(" = "); - sb.append( - Randomly.fromOptions( - StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType - .getRandomValue(table.getColumns().get(i).getType().getPrimitiveDataType())), - "DEFAULT")); + sb.append(Randomly.fromOptions( + StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType.getRandomValue( + table.getColumns().get(i).getType().getPrimitiveDataType(), globalState.getRandomly())), + "DEFAULT")); } } From 5839a787f6168b9baea2a1f6cf58fb4b7d05d978 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:22:07 +0800 Subject: [PATCH 376/856] refactor: refactor stonedb aggregate function (#874) --- .../stonedb/ast/StoneDBAggregate.java | 34 +++++++++++++++++++ .../gen/StoneDBExpressionGenerator.java | 20 +---------- .../oracle/StoneDBAggregateOracle.java | 2 +- 3 files changed, 36 insertions(+), 20 deletions(-) create mode 100644 src/sqlancer/stonedb/ast/StoneDBAggregate.java diff --git a/src/sqlancer/stonedb/ast/StoneDBAggregate.java b/src/sqlancer/stonedb/ast/StoneDBAggregate.java new file mode 100644 index 000000000..2e8fb3bc0 --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBAggregate.java @@ -0,0 +1,34 @@ +package sqlancer.stonedb.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; + +public class StoneDBAggregate extends FunctionNode + implements StoneDBExpression { + + // https://stonedb.io/docs/SQL-reference/functions/aggregate-functions/ + public enum StoneDBAggregateFunction { + MAX(1), MIN(1), AVG(1), COUNT(1), FIRST(1), SUM(1); + + private int nrArgs; + + StoneDBAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static StoneDBAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + protected StoneDBAggregate(StoneDBAggregateFunction function, List args) { + super(function, args); + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 6ed9d59bb..36adf85f2 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -26,6 +26,7 @@ import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -180,25 +181,6 @@ protected List> generateColumns() { return new ArrayList<>(set); } - // https://stonedb.io/docs/SQL-reference/functions/aggregate-functions/ - public enum StoneDBAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1), FIRST(1), SUM(1); - - private int nrArgs; - - StoneDBAggregateFunction(int nrArgs) { - this.nrArgs = nrArgs; - } - - public static StoneDBAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - } - public enum StoneDBUnaryPrefixOperator implements Operator { NOT("NOT"), PLUS("+"), MINUS("-"), INVERSION("!"); diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java index df09e8961..652fa30ed 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -19,9 +19,9 @@ import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; import sqlancer.stonedb.ast.StoneDBExpression; import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBAggregateFunction; import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPostfixOperator; import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPrefixOperator; From ec8c8b03cd44a0567400ae0d8e9ad994c4e2092d Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:22:35 +0800 Subject: [PATCH 377/856] feat: stonedb view (#872) --- src/sqlancer/stonedb/StoneDBProvider.java | 10 ++- .../gen/StoneDBViewCreateGenerator.java | 78 +++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java index 55b7aff70..e4d9c46f8 100644 --- a/src/sqlancer/stonedb/StoneDBProvider.java +++ b/src/sqlancer/stonedb/StoneDBProvider.java @@ -28,6 +28,7 @@ import sqlancer.stonedb.gen.StoneDBTableDeleteGenerator; import sqlancer.stonedb.gen.StoneDBTableInsertGenerator; import sqlancer.stonedb.gen.StoneDBTableUpdateGenerator; +import sqlancer.stonedb.gen.StoneDBViewCreateGenerator; @AutoService(DatabaseProvider.class) public class StoneDBProvider extends SQLProviderAdapter { @@ -54,7 +55,12 @@ enum Action implements AbstractAction { TABLE_UPDATE(StoneDBTableUpdateGenerator::generate), // INDEX_CREATE(StoneDBIndexCreateGenerator::generate), // INDEX_DROP(StoneDBIndexDropGenerator::generate), // - TABLE_INSERT(StoneDBTableInsertGenerator::generate); // + TABLE_INSERT(StoneDBTableInsertGenerator::generate), // + + VIEW_CREATE((g) -> { + String viewName = g.getSchema().getFreeViewName(); + return StoneDBViewCreateGenerator.generate(g, viewName); + }); private final SQLQueryProvider sqlQueryProvider; @@ -90,6 +96,8 @@ private static int mapActions(StoneDBGlobalState globalState, Action a) { return r.getInteger(0, 1); case INDEX_DROP: return r.getInteger(0, 1); + case VIEW_CREATE: + return r.getInteger(0, 1); default: throw new AssertionError(a); } diff --git a/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java new file mode 100644 index 000000000..3ee25ffbd --- /dev/null +++ b/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java @@ -0,0 +1,78 @@ +package sqlancer.stonedb.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBSchema.StoneDBTables; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; + +public class StoneDBViewCreateGenerator { + // the name of the view to create + private final String viewName; + private final StoneDBGlobalState globalState; + private StoneDBSelect select; + private final StringBuilder sb = new StringBuilder(); + + public StoneDBViewCreateGenerator(StoneDBGlobalState globalState, String viewName) { + this.globalState = globalState; + this.viewName = viewName; + setSelect(); + } + + public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String viewName) { + return new StoneDBViewCreateGenerator(globalState, viewName).getQuery(); + } + + private SQLQueryAdapter getQuery() { + sb.append("CREATE "); + sb.append("VIEW "); + sb.append(viewName); + sb.append(" AS "); + sb.append(StoneDBToStringVisitor.asString(select)); + return new SQLQueryAdapter(sb.toString(), true); + } + + private void setSelect() { + StoneDBSchema schema = globalState.getSchema(); + StoneDBTables targetTables = schema.getRandomTableNonEmptyTables(); + StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); + + select = new StoneDBSelect(); + select.setFetchColumns(generateFetchColumns(targetTables)); + List tables = targetTables.getTables(); + List> tableReferenceNodeList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = StoneDBJoin.getJoins(tableReferenceNodeList, globalState); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableReferenceNodeList)); + select.setWhereClause(gen.generateExpression()); + select.setOrderByExpressions(gen.generateOrderBys()); + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + + private List> generateFetchColumns(StoneDBTables tables) { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))); + } else { + columns = Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + } + return columns; + } +} From c7c0d7683399475f67912bc799a111dd773abd6a Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:24:20 +0800 Subject: [PATCH 378/856] fix: syntax error (#871) --- .../stonedb/gen/StoneDBTableAlterGenerator.java | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index 0510f7a78..64ddefa1b 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -1,6 +1,5 @@ package sqlancer.stonedb.gen; -import java.util.List; import java.util.regex.Pattern; import sqlancer.Randomly; @@ -34,7 +33,7 @@ private SQLQueryAdapter getQuery() { sb.append("ALTER TABLE "); sb.append(table.getName()); sb.append(" "); - appendAlterOptions(); + appendAlterOption(Randomly.fromOptions(Action.values())); addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors, true); } @@ -56,19 +55,6 @@ private void addExpectedErrors() { .compile("Column length too big for column 'c\\d{1,3}' \\(max = 16383\\); use BLOB or TEXT instead")); } - private void appendAlterOptions() { - List actions; - if (Randomly.getBooleanWithSmallProbability()) { - actions = Randomly.subset(Action.values()); - } else { - actions = List.of(Randomly.fromOptions(Action.values())); - } - for (Action action : actions) { - appendAlterOption(action); - sb.append(" "); - } - } - private void appendAlterOption(Action action) { StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) .setColumns(table.getColumns()); From 290288235e49a53de9e7663fc59d44d7bbdd4787 Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Thu, 10 Aug 2023 15:39:53 +0800 Subject: [PATCH 379/856] feat: write log when running statement reducer (#864) * feat: write log when running statement reducer + write log when start reducing, finish reducing and when a reduction step is successfully performed + fix some bugs of the statement reducer. * fix: check upper bound of subList to avoid outOfBoundException * overwrite log content of reducer * feat: create symbolic link to the latest reduce log * fix format issues * fix: log reduced result overwritting previous steps --- src/sqlancer/Main.java | 53 ++++++++++++++++- src/sqlancer/StatementReducer.java | 58 +++++++++++++------ test/sqlancer/reducer/TestEnvironment.java | 10 +++- .../reducer/TestStatementReducer.java | 7 +++ 4 files changed, 106 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 76d7a36f4..70657c3d6 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -76,12 +76,16 @@ public static final class StateLogger { private final File loggerFile; private File curFile; private File queryPlanFile; + private File reduceFile; private FileWriter logFileWriter; public FileWriter currentFileWriter; private FileWriter queryPlanFileWriter; + private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; private final boolean logQueryPlan; + + private final boolean useReducer; private final DatabaseProvider databaseProvider; private static final class AlsoWriteToConsoleFileWriter extends FileWriter { @@ -118,6 +122,15 @@ public StateLogger(String databaseName, DatabaseProvider provider, Main if (logQueryPlan) { queryPlanFile = new File(dir, databaseName + "-plan.log"); } + this.useReducer = options.useReducer(); + if (useReducer) { + File reduceFileDir = new File(dir, "reduce"); + if (!reduceFileDir.exists()) { + reduceFileDir.mkdir(); + } + this.reduceFile = new File(reduceFileDir, databaseName + "-reduce.log"); + + } this.databaseProvider = provider; } @@ -183,6 +196,20 @@ public FileWriter getQueryPlanFileWriter() { return queryPlanFileWriter; } + public FileWriter getReduceFileWriter() { + if (!useReducer) { + throw new UnsupportedOperationException(); + } + FileWriter fileWriter; + try { + fileWriter = new FileWriter(reduceFile, false); + } catch (IOException e) { + throw new AssertionError(e); + } + + return fileWriter; + } + public void writeCurrent(StateToReproduce state) { if (!logEachSelect) { throw new UnsupportedOperationException(); @@ -229,6 +256,30 @@ public void writeQueryPlan(String queryPlan) { } } + public void logReduced(StateToReproduce state) { + FileWriter reduceFileWriter = getReduceFileWriter(); + + StringBuilder sb = new StringBuilder(); + for (Query s : state.getStatements()) { + sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); + } + try { + reduceFileWriter.write(sb.toString()); + + } catch (IOException e) { + throw new AssertionError(e); + } finally { + try { + reduceFileWriter.flush(); + reduceFileWriter.close(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + + } + public void logException(Throwable reduce, StateToReproduce state) { Loggable stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); @@ -395,7 +446,7 @@ public void run() throws Exception { } if (reproducer != null && options.useReducer()) { System.out.println("EXPERIMENTAL: Trying to reduce queries using a simple reducer."); - System.out.println("Reduced query will be output to stdout but not logs."); + // System.out.println("Reduced query will be output to stdout but not logs."); G newGlobalState = createGlobalState(); newGlobalState.setState(stateToRepro); newGlobalState.setRandomly(r); diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java index 3d3d36711..e066aca84 100644 --- a/src/sqlancer/StatementReducer.java +++ b/src/sqlancer/StatementReducer.java @@ -13,6 +13,14 @@ public class StatementReducer, O extends DBMSSpec private boolean observedChange; private int partitionNum; + private long currentReduceSteps; + private long currentReduceTime; + + private long maxReduceSteps; + private long maxReduceTime; + + Instant timeOfReductionBegins; + public StatementReducer(DatabaseProvider provider) { this.provider = provider; } @@ -28,25 +36,26 @@ private boolean hasNotReachedLimit(long curr, long limit) { @Override public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { - long maxReduceTime = state.getOptions().getMaxStatementReduceTime(); - long maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); + maxReduceTime = state.getOptions().getMaxStatementReduceTime(); + maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); List> knownToReproduceBugStatements = new ArrayList<>(); for (Query stat : state.getState().getStatements()) { knownToReproduceBugStatements.add((Query) stat); } - System.out.println("Starting query:"); - printQueries(knownToReproduceBugStatements); - System.out.println(); + // System.out.println("Starting query:"); + // Main.StateLogger logger = newGlobalState.getLogger(); + // printQueries(knownToReproduceBugStatements); + // System.out.println(); if (knownToReproduceBugStatements.size() <= 1) { return; } - Instant timeOfReductionBegins = Instant.now(); - long currentReduceSteps = 0; - long currentReduceTime = 0; + timeOfReductionBegins = Instant.now(); + currentReduceSteps = 0; + currentReduceTime = 0; partitionNum = 2; while (knownToReproduceBugStatements.size() >= 2 && hasNotReachedLimit(currentReduceSteps, maxReduceSteps) @@ -55,23 +64,21 @@ && hasNotReachedLimit(currentReduceTime, maxReduceTime)) { knownToReproduceBugStatements = tryReduction(state, reproducer, newGlobalState, knownToReproduceBugStatements); + if (!observedChange) { if (partitionNum == knownToReproduceBugStatements.size()) { break; } // increase the search granularity partitionNum = Math.min(partitionNum * 2, knownToReproduceBugStatements.size()); - - currentReduceSteps++; - Instant currentInstant = Instant.now(); - currentReduceTime = Duration.between(currentInstant, timeOfReductionBegins).getSeconds(); } - } - System.out.println("Reduced query:"); - printQueries(knownToReproduceBugStatements); + // System.out.println("Reduced query:"); + // printQueries(knownToReproduceBugStatements); newGlobalState.getState().setStatements(new ArrayList<>(knownToReproduceBugStatements)); + newGlobalState.getLogger().logReduced(newGlobalState.getState()); + } private List> tryReduction(G state, // NOPMD @@ -87,7 +94,8 @@ private List> tryReduction(G state, // NOPMD try (C con2 = provider.createDatabase(newGlobalState)) { newGlobalState.setConnection(con2); List> candidateStatements = new ArrayList<>(statements); - candidateStatements.subList(start, start + subLength).clear(); + int endPoint = Math.min(start + subLength, candidateStatements.size()); + candidateStatements.subList(start, endPoint).clear(); newGlobalState.getState().setStatements(new ArrayList<>(candidateStatements)); for (Query s : candidateStatements) { @@ -102,19 +110,32 @@ private List> tryReduction(G state, // NOPMD observedChange = true; statements = candidateStatements; partitionNum = Math.max(partitionNum - 1, 2); - break; // reproducer.outputHook((SQLite3GlobalState) newGlobalState); - // state.getLogger().logReduced(newGlobalState.getState()); + newGlobalState.getLogger().logReduced(newGlobalState.getState()); + break; + } } catch (Throwable ignoredException) { } + } catch (Exception e) { + e.printStackTrace(); + } + + currentReduceSteps++; + Instant currentInstant = Instant.now(); + + currentReduceTime = Duration.between(timeOfReductionBegins, currentInstant).getSeconds(); + if (!hasNotReachedLimit(currentReduceSteps, maxReduceSteps) + || !hasNotReachedLimit(currentReduceTime, maxReduceTime)) { + return statements; } start = start + subLength; } return statements; } + @SuppressWarnings("unused") private void printQueries(List> statements) { System.out.println("==============================="); for (Query q : statements) { @@ -122,5 +143,4 @@ private void printQueries(List> statements) { } System.out.println("==============================="); } - } diff --git a/test/sqlancer/reducer/TestEnvironment.java b/test/sqlancer/reducer/TestEnvironment.java index 3ba1fc11e..7c1155490 100644 --- a/test/sqlancer/reducer/TestEnvironment.java +++ b/test/sqlancer/reducer/TestEnvironment.java @@ -6,6 +6,7 @@ import sqlancer.reducer.VirtualDB.VirtualDBProvider; import sqlancer.reducer.VirtualDB.VirtualDBQuery; +import java.lang.reflect.Field; import java.util.ArrayList; import java.util.List; import java.util.ServiceLoader; @@ -88,6 +89,10 @@ private void setUpTestingEnvironment() throws Exception { state.setState(stateToReproduce); state.setDatabaseName(databaseName); + // A really hacky way to enable reducer... + Field field = options.getClass().getDeclaredField("useReducer"); + field.setAccessible(true); + field.set(options, true); state.setMainOptions(options); // Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); @@ -96,8 +101,9 @@ private void setUpTestingEnvironment() throws Exception { try (SQLConnection con = provider.createDatabase(state)) { state.setConnection(con); newGlobalState = createGlobalState(); - // Main.StateLogger newLogger = new Main.StateLogger(databaseName, provider, options); - // newGlobalState.setStateLogger(newLogger); + Main.StateLogger newLogger = new Main.StateLogger(databaseName, provider, options); + newGlobalState.setStateLogger(newLogger); + state.setStateLogger(newLogger); newGlobalState.setState(stateToReproduce); newGlobalState.setDatabaseName(databaseName); newGlobalState.setMainOptions(options); diff --git a/test/sqlancer/reducer/TestStatementReducer.java b/test/sqlancer/reducer/TestStatementReducer.java index 66a9a3c4c..d9e364c84 100644 --- a/test/sqlancer/reducer/TestStatementReducer.java +++ b/test/sqlancer/reducer/TestStatementReducer.java @@ -1,6 +1,7 @@ package sqlancer.reducer; import org.junit.jupiter.api.Test; +import sqlancer.Main; import sqlancer.common.query.Query; import java.util.ArrayList; @@ -72,4 +73,10 @@ void testDeltaDebuggingWithStatementsCombination() throws Exception { assertEquals(queriesString, "Statement_2;\nStatement_318;\nStatement_990;"); } + @Test + void testSQLite3WithStatementReducer() { + Main.executeMain(new String[] { "--random-seed", "0", "--use-reducer", "--timeout-seconds", "60", + "--num-threads", "4", "sqlite3", "--oracle", "NoREC" }); + } + } From b6cf5c03e474ed171136c0789d8aa8b3efc6e2b6 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Fri, 11 Aug 2023 10:28:19 +0800 Subject: [PATCH 380/856] feat: add expected errors for stonedb (#867) * feat: add expected errors * feat: add expected errors --- src/sqlancer/stonedb/StoneDBErrors.java | 29 +++++++++++++++++++ .../gen/StoneDBTableAlterGenerator.java | 15 ++++++++++ .../gen/StoneDBTableDeleteGenerator.java | 6 ++++ .../gen/StoneDBTableInsertGenerator.java | 4 +++ .../oracle/StoneDBAggregateOracle.java | 2 ++ .../stonedb/oracle/StoneDBNoRECOracle.java | 2 ++ 6 files changed, 58 insertions(+) diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java index bdbd6a447..1e72e5685 100644 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -1,6 +1,35 @@ package sqlancer.stonedb; +import sqlancer.common.query.ExpectedErrors; + public final class StoneDBErrors { private StoneDBErrors() { } + + public static void addExpectedExpressionErrors(ExpectedErrors errors) { + // java.sql.SQLException: Incorrect DATE value: '292269055-12-02' + errors.add("Incorrect DATE value: "); + // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 + errors.add("Incorrect string value: "); + // java.sql.SQLException: Incorrect integer value: 'ST' for column 'c1' at row 1 + errors.add("Incorrect integer value: "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Truncated incorrect INTEGER value: '#Q' + errors.add("Data truncation: Truncated incorrect INTEGER value: "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT value is out of range in + // '-((`database0`.`t0`.`c1` >> (not(`database0`.`t0`.`c1`))))' + errors.add("Data truncation: BIGINT value is out of range in "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT UNSIGNED value is out of range in + // '(`database10`.`t0`.`c0` + (`database10`.`t0`.`c0` & (not(0.5))))' + errors.add("Data truncation: BIGINT UNSIGNED value is out of range in "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect time value: '0Sly6xqF0' for + // column 'c1' at row 1 + errors.add("Data truncation: Incorrect time value: "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: DOUBLE value is out of range in + // '(`database0`.`t0`.`c0` * `database0`.`t0`.`c0`)' + errors.add("Data truncation: DOUBLE value is out of range in "); + // java.sql.SQLException: Numeric result of an expression is too large and cannot be handled by tianmu. + errors.add("Numeric result of an expression is too large and cannot be handled by tianmu."); + // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order clause' + errors.add("Unknown column "); + } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index 64ddefa1b..e9c283cc9 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -41,6 +41,14 @@ private SQLQueryAdapter getQuery() { private void addExpectedErrors() { // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Data too long for column 'c0' at row 2 errors.addRegex(Pattern.compile("Data truncation: Data too long for column 'c\\d{1,3}' at row \\d{1,3}")); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: + // '0.571272522740968' for column 'c1' at row 1 + errors.add("Incorrect datetime value: "); + // java.sql.SQLSyntaxErrorException: Invalid default value for 'c0' + errors.add("Invalid default value for "); + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row + // 2 + errors.add("Data truncation: Out of range value for column "); // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes errors.add("Specified key was too long; max key length is 3072 bytes"); // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead @@ -98,12 +106,19 @@ private void appendAlterOption(Action action) { String oldColumnName = table.getRandomColumn().getName(); String newColumnName = table.getFreeColumnName(); sb.append(oldColumnName).append(" ").append(newColumnName).append(" "); + errors.add("Incorrect integer value: "); + // java.sql.SQLException: Data truncated for column 'c1' at row 1 + errors.add("Data truncated for column "); sb.append( StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT // instead errors.addRegex(Pattern .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); + // java.sql.SQLSyntaxErrorException: BLOB column 'c1' can't be used in key specification with the used table + // type + errors.addRegex(Pattern + .compile("BLOB column 'c\\d{1,3}' can't be used in key specification with the used table type")); if (Randomly.getBoolean()) { if (Randomly.getBoolean()) { sb.append(" FIRST"); diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index ab0828d30..16b9c1061 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -55,6 +55,12 @@ public SQLQueryAdapter getQuery() { sb.append(" LIMIT "); sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); } + addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } + + private void addExpectedErrors() { + // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 + errors.add("Incorrect string value: "); + } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index f4435ec26..19b650b82 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -53,6 +53,10 @@ private SQLQueryAdapter getQuery() { } private void addExpectedErrors() { + // java.sql.SQLException: Incorrect DATE value: '292278994-08-17' + errors.add("Incorrect DATE value: '"); + // java.sql.SQLIntegrityConstraintViolationException: Duplicate entry '1970-01-14' for key 'PRIMARY' + errors.add("Duplicate entry "); // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row errors.add("Data truncation: Out of range value for column "); // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 'field list' diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java index 652fa30ed..27ba91013 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -15,6 +15,7 @@ import sqlancer.common.ast.newast.Node; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.stonedb.StoneDBErrors; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; @@ -30,6 +31,7 @@ public class StoneDBAggregateOracle extends StoneDBQueryPartitioningBase { public StoneDBAggregateOracle(StoneDBGlobalState state) { super(state); + StoneDBErrors.addExpectedExpressionErrors(errors); } @Override diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 534bf030d..e042cd44a 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -21,6 +21,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.stonedb.StoneDBErrors; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; @@ -42,6 +43,7 @@ public class StoneDBNoRECOracle extends NoRECBase implements public StoneDBNoRECOracle(StoneDBGlobalState globalState) { super(globalState); this.schema = globalState.getSchema(); + StoneDBErrors.addExpectedExpressionErrors(errors); } @Override From 0009112ada915f120bb5b2209455d3794093d939 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Sat, 12 Aug 2023 10:05:10 +0800 Subject: [PATCH 381/856] fix: fix implementation bugs (#876) * fix: drop features StoneDB not supported * fix: add supported operator * git: merge latest changes --- src/sqlancer/stonedb/ast/StoneDBAggregate.java | 2 +- src/sqlancer/stonedb/ast/StoneDBJoin.java | 2 +- src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/stonedb/ast/StoneDBAggregate.java b/src/sqlancer/stonedb/ast/StoneDBAggregate.java index 2e8fb3bc0..f65b54d74 100644 --- a/src/sqlancer/stonedb/ast/StoneDBAggregate.java +++ b/src/sqlancer/stonedb/ast/StoneDBAggregate.java @@ -11,7 +11,7 @@ public class StoneDBAggregate extends FunctionNode Date: Mon, 14 Aug 2023 12:07:50 +0800 Subject: [PATCH 382/856] feat: bypass logical XOR (#877) * feat: bypass logical XOR * fix: change import order * fix: fix style check error --- src/sqlancer/stonedb/StoneDBBugs.java | 6 ++++++ src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java | 9 ++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java index 11a144dad..4864c56e5 100644 --- a/src/sqlancer/stonedb/StoneDBBugs.java +++ b/src/sqlancer/stonedb/StoneDBBugs.java @@ -39,6 +39,12 @@ public final class StoneDBBugs { // INSERT INTO t0(c0) VALUE ('16'); // SELECT t0.c0 FROM t0 WHERE t0.c0; -- expected 1 row but got 2 rows public static boolean bugNotReported7 = true; + // CREATE TABLE t0(c0 INT) ; + // SELECT * FROM t0 WHERE ((t0.c0)XOR((t0.c0))); + // ERROR 6 (HY000): The query includes syntax that is not supported by the storage engine. Either restructure the + // query with supported syntax, or enable the MySQL core::Query Path in config file to execute the query with + // reduced performance. + public static boolean bugNotReportedXOR = true; private StoneDBBugs() { } diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 36adf85f2..6ff9f1b5b 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -3,12 +3,14 @@ import static sqlancer.stonedb.StoneDBBugs.bug1942; import static sqlancer.stonedb.StoneDBBugs.bugNotReported3; import static sqlancer.stonedb.StoneDBBugs.bugNotReported6; +import static sqlancer.stonedb.StoneDBBugs.bugNotReportedXOR; import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -146,7 +148,12 @@ protected Node generateExpression(int depth) { return new NewInOperatorNode<>(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), true); case BINARY_LOGICAL: - op = StoneDBBinaryLogicalOperator.getRandom(); + if (bugNotReportedXOR) { + op = Randomly.fromList(Arrays.stream(StoneDBBinaryLogicalOperator.values()) + .filter(p -> p != StoneDBBinaryLogicalOperator.XOR).collect(Collectors.toList())); + } else { + op = StoneDBBinaryLogicalOperator.getRandom(); + } return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_ARITHMETIC: op = StoneDBBinaryArithmeticOperator.getRandom(); From 9bcc3ffe819e87e80fe9cd2e0d1f7058501059ee Mon Sep 17 00:00:00 2001 From: Jiansen Song Date: Mon, 14 Aug 2023 12:08:30 +0800 Subject: [PATCH 383/856] fix: avoid logging CREATE TABLE exception (#875) * fix: avoid logging CREATE TABLE exception * Update TestLoggableFactory.java * Update SQLLoggableFactory.java We need to check the query in the constructor * Update SQLLoggableFactory.java Fix wrong indent. * fix: check couldAffectSchema before setting false * fix: revise check query string --------- Co-authored-by: Yichen Yan --- src/sqlancer/common/query/SQLQueryAdapter.java | 12 +++++++++--- test/sqlancer/TestLoggableFactory.java | 17 +++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) create mode 100644 test/sqlancer/TestLoggableFactory.java diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 22eef268a..8e698676a 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -24,7 +24,11 @@ public SQLQueryAdapter(String query, boolean couldAffectSchema) { } public SQLQueryAdapter(String query, ExpectedErrors expectedErrors) { - this(query, expectedErrors, false); + this(query, expectedErrors, guessAffectSchemaFromQuery(query)); + } + + private static boolean guessAffectSchemaFromQuery(String query) { + return query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN"); } public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema) { @@ -46,8 +50,10 @@ private String canonicalizeString(String s) { } private void checkQueryString() { - if (query.contains("CREATE TABLE") && !query.startsWith("EXPLAIN") && !couldAffectSchema) { - throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); + if (!couldAffectSchema) { + if (guessAffectSchemaFromQuery(query)) { + throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); + } } } diff --git a/test/sqlancer/TestLoggableFactory.java b/test/sqlancer/TestLoggableFactory.java new file mode 100644 index 000000000..2679cf740 --- /dev/null +++ b/test/sqlancer/TestLoggableFactory.java @@ -0,0 +1,17 @@ +package sqlancer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.log.SQLLoggableFactory; +import sqlancer.common.query.SQLQueryAdapter; + +public class TestLoggableFactory { + + @Test + public void testLogCreateTable() { + String query = "CREATE TABLE t1 (c1 INT)"; + SQLLoggableFactory logger = new SQLLoggableFactory(); + SQLQueryAdapter queryAdapter = logger.getQueryForStateToReproduce(query); + assert (queryAdapter.couldAffectSchema()); + } + +} From b694a3abda768ab9f562067d57f15a0a5b9db3b1 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 22 Aug 2023 23:33:23 +0800 Subject: [PATCH 384/856] feat: add support of insert on duplicate key update (#883) --- src/sqlancer/common/query/SQLQueryAdapter.java | 6 ++---- .../stonedb/gen/StoneDBTableInsertGenerator.java | 9 +++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 8e698676a..248403458 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -50,10 +50,8 @@ private String canonicalizeString(String s) { } private void checkQueryString() { - if (!couldAffectSchema) { - if (guessAffectSchemaFromQuery(query)) { - throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); - } + if (!couldAffectSchema && guessAffectSchemaFromQuery(query)) { + throw new AssertionError("CREATE TABLE statements should set couldAffectSchema to true"); } } diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index 19b650b82..a9823be6d 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -48,6 +48,7 @@ private SQLQueryAdapter getQuery() { sb.append(table.getName()); appendPartition(); appendColumnsAndValues(columns); + appendOnDuplicateUpdate(); addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); } @@ -83,6 +84,14 @@ private void appendColumnsAndValues(List columns) { appendValues(); } + private void appendOnDuplicateUpdate() { + sb.append("on duplicate key update "); + StoneDBColumn randomColumn = table.getRandomColumn(); + sb.append(randomColumn.getName()); + sb.append("="); + insertValue(randomColumn); + } + // append nrRows rows private void appendValues() { int nrRows; From 7f37778cae1064c03f8c4721fbe55d88253996d5 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 22 Aug 2023 23:34:12 +0800 Subject: [PATCH 385/856] feat: add support of advanced functions (#882) --- .../stonedb/ast/StoneDBAdvancedFunction.java | 34 +++++++++++++++++++ .../gen/StoneDBExpressionGenerator.java | 6 ++++ 2 files changed, 40 insertions(+) create mode 100644 src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java diff --git a/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java b/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java new file mode 100644 index 000000000..9c49f2678 --- /dev/null +++ b/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java @@ -0,0 +1,34 @@ +package sqlancer.stonedb.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.stonedb.ast.StoneDBAdvancedFunction.StoneDBAdvancedFunc; + +public class StoneDBAdvancedFunction extends FunctionNode + implements StoneDBExpression { + + // https://stonedb.io/docs/SQL-reference/functions/advanced-functions + public enum StoneDBAdvancedFunc { + IFNULL(2), IF(3), NULLIF(2), BIN(1), BINARY(1), CONV(3); + + private int nrArgs; + + StoneDBAdvancedFunc(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static StoneDBAdvancedFunc getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + protected StoneDBAdvancedFunction(StoneDBAdvancedFunc function, List args) { + super(function, args); + } +} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java index 6ff9f1b5b..3e1dcd348 100644 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java @@ -28,6 +28,7 @@ import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; +import sqlancer.stonedb.ast.StoneDBAdvancedFunction.StoneDBAdvancedFunc; import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; import sqlancer.stonedb.ast.StoneDBConstant; import sqlancer.stonedb.ast.StoneDBExpression; @@ -118,6 +119,11 @@ protected Node generateExpression(int depth) { return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs(), depth + 1), aggregateFunction); } + if (Randomly.getBooleanWithRatherLowProbability()) { + StoneDBAdvancedFunc advancedFunction = StoneDBAdvancedFunc.getRandom(); + return new NewFunctionNode<>(generateExpressions(advancedFunction.getNrArgs(), depth + 1), + advancedFunction); + } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); Expression expr = Randomly.fromList(possibleOptions); Operator op; From 11d57fde3d4870ebfadf4f1417c503803005dec3 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Wed, 23 Aug 2023 10:12:50 +0800 Subject: [PATCH 386/856] feat: add support of Fuzz oracle (#869) * feat: add expected ExpressionErrors feat: add Fuzz oracle feat: add expected errors feat: add one test oracle * ignore: let CI run * feat: generate random columns * fix: remove redundant select * fix: change enum name * fix: change enum name --- .github/workflows/main.yml | 31 +-------- src/sqlancer/stonedb/StoneDBOptions.java | 7 ++ .../stonedb/oracle/StoneDBFuzzOracle.java | 67 +++++++++++++++++++ test/sqlancer/dbms/TestStoneDBFuzz.java | 21 ++++++ 4 files changed, 98 insertions(+), 28 deletions(-) create mode 100644 src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java create mode 100644 test/sqlancer/dbms/TestStoneDBFuzz.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9aaed3c70..ec8698f03 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -375,8 +375,8 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test - stonedb-norec: - name: DBMS Tests (StoneDB NoRec) + stonedb: + name: DBMS Tests (StoneDB) runs-on: ubuntu-latest steps: @@ -391,32 +391,7 @@ jobs: sleep 60s docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 - with: - java-version: 11 - - name: Build - run: mvn -B package -DskipTests=true - - name: StoneDB Tests - run: | - STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBNoRec - - stonedb-tlp: - name: DBMS Tests (StoneDB TLP) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up StoneDB - run: | - docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb - ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb --format "{{.ID}}") - docker exec $ContainerID bash sleep 60s - docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf - docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Set up JDK 11 uses: actions/setup-java@v1.4.4 with: @@ -425,7 +400,7 @@ jobs: run: mvn -B package -DskipTests=true - name: StoneDB Tests run: | - STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBTLP + STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBFuzz tidb: name: DBMS Tests (TiDB) diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java index 7e6a2f09b..a9728323f 100644 --- a/src/sqlancer/stonedb/StoneDBOptions.java +++ b/src/sqlancer/stonedb/StoneDBOptions.java @@ -14,6 +14,7 @@ import sqlancer.stonedb.StoneDBOptions.StoneDBOracleFactory; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; import sqlancer.stonedb.oracle.StoneDBAggregateOracle; +import sqlancer.stonedb.oracle.StoneDBFuzzOracle; import sqlancer.stonedb.oracle.StoneDBNoRECOracle; import sqlancer.stonedb.oracle.StoneDBQueryPartitioningDistinctTester; import sqlancer.stonedb.oracle.StoneDBQueryPartitioningGroupByTester; @@ -30,6 +31,12 @@ public class StoneDBOptions implements DBMSSpecificOptions public List oracles = List.of(StoneDBOracleFactory.NOREC); public enum StoneDBOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(StoneDBGlobalState globalState) throws SQLException { + return new StoneDBFuzzOracle(globalState); + } + }, NOREC { @Override public TestOracle create(StoneDBGlobalState globalState) throws SQLException { diff --git a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java new file mode 100644 index 000000000..7e4d6a622 --- /dev/null +++ b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java @@ -0,0 +1,67 @@ +package sqlancer.stonedb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.stonedb.StoneDBErrors; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; +import sqlancer.stonedb.StoneDBSchema; +import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; +import sqlancer.stonedb.StoneDBSchema.StoneDBTable; +import sqlancer.stonedb.StoneDBSchema.StoneDBTables; +import sqlancer.stonedb.StoneDBToStringVisitor; +import sqlancer.stonedb.ast.StoneDBExpression; +import sqlancer.stonedb.ast.StoneDBJoin; +import sqlancer.stonedb.ast.StoneDBSelect; +import sqlancer.stonedb.gen.StoneDBExpressionGenerator; + +public class StoneDBFuzzOracle implements TestOracle { + private final StoneDBGlobalState globalState; + private final StoneDBSchema schema; + private final ExpectedErrors errors = new ExpectedErrors(); + + public StoneDBFuzzOracle(StoneDBGlobalState globalState) { + this.globalState = globalState; + this.schema = globalState.getSchema(); + StoneDBErrors.addExpectedExpressionErrors(errors); + } + + @Override + public void check() throws Exception { + StoneDBTables randomTables = schema.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(globalState).setColumns(columns); + Node randomWhereCondition = gen.generateExpression(); + List tables = randomTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List> joins = StoneDBJoin.getJoins(tableList, globalState); + + StoneDBSelect select = new StoneDBSelect(); + select.setFetchColumns( + Randomly.fromOptions(List.of(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))), + allColumns, Randomly.subset(allColumns))); + select.setFromList(new ArrayList<>(tableList)); + select.setJoinList(joins); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions( + new StoneDBExpressionGenerator(globalState).setColumns(columns).generateOrderBys()); + } + select.setJoinList(joins); + + String queryString = StoneDBToStringVisitor.asString(select); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + q.executeAndGetLogged(globalState); + } +} diff --git a/test/sqlancer/dbms/TestStoneDBFuzz.java b/test/sqlancer/dbms/TestStoneDBFuzz.java new file mode 100644 index 000000000..2f5d52424 --- /dev/null +++ b/test/sqlancer/dbms/TestStoneDBFuzz.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestStoneDBFuzz { + + String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); + boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); + + @Test + public void testStoneDB() { + assumeTrue(stoneDBIsAvailable); + assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", + "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "FUZZER")); + } + +} \ No newline at end of file From 48aadfd27a889bb7544db6cbd18ad4608606b3bb Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Wed, 23 Aug 2023 23:04:43 +0800 Subject: [PATCH 387/856] add doc of test case reduction (#880) * add doc of test case reduction * made some improvement --- docs/testCaseReduction.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 docs/testCaseReduction.md diff --git a/docs/testCaseReduction.md b/docs/testCaseReduction.md new file mode 100644 index 000000000..35aa9cd08 --- /dev/null +++ b/docs/testCaseReduction.md @@ -0,0 +1,31 @@ +# Test Case Reduction +SQLancer generates a large number of statements, but not all of them are relevant to the bug. To automatically reduce the test cases, two reducers were implemented: the statement reducer and the AST-based reducer. + +## Statement Reducer +The statement reducer utilizes the delta-debugging technique to remove irrelevant statements. More details of delta-debugging could be found in this paper: [Simplifying and Isolating Failure-Inducing Input](https://www.cs.purdue.edu/homes/xyzhang/fall07/Papers/delta-debugging.pdf). + +Using the statement reducer, SQLancer reduces the set of statements to a minimal subset that reproduces the bug. + +## AST-Based Reducer +The AST-based reducer can shorten a statement by applying AST level transformations, including removing unnecessary clauses, irrelevant elements in a list, simplify complicated expressions and etc. + +The transformations are implemented by [JSQLParser](https://github.com/JSQLParser/JSqlParser), a RDBMS agnostic SQL statement parser that can translate SQL statements into a traversable hierarchy of Java classes. JSQLParser provides support for the SQL standard as well as major SQL dialects. The AST-based reducer works for any SQL dialects that can be parsed by this tool. + +## Enable reducers +Test-case reduction is disabled by default. The statement reducer can be enabled by passing `--use-reducer` when starting SQLancer. If you wish to further shorten each statements, you need to additionally pass the `--reduce-AST` parameter so that the AST-based reduction is applied. + +Note: if `--reduce-AST` is set, `--use-reducer` option must be enabled first. + +There are also options to define timeout seconds and max steps of reduction for both statement reducer and AST-based reducer. + +``` +--statement-reducer-max-steps= +--statement-reducer-max-time= +--ast-reducer-max-steps= +--ast-reducer-max-time= +``` + +## Reduction logs +If test-case reduction is enabled, each time the reducer performs a reduction step successfully,it prints the reduced statements to the log file, overwriting the previous ones. + +The log files will be stored in the following format: `logs//reduce/-reduce.log`. For instance, if the tested DBMS is SQLite3 and the current database is named database0, the log file will be located at `logs/sqlite3/reduce/database0-reduce.log`. \ No newline at end of file From 706ac16bc554bf096ee049521960f1beff40e212 Mon Sep 17 00:00:00 2001 From: Yutan Yang <127477377+ColinYoungTaro@users.noreply.github.com> Date: Thu, 24 Aug 2023 16:35:21 +0800 Subject: [PATCH 388/856] feat: refactor AST-based reducer (#879) * feat: refactor AST-based reducer + define different kinds of transformers separately, making the reducer extenable. + add options to enable AST Based Reducer * modify the code based on the feedback of the review * add expected value to test cases for AST-based reducer. * support removing columns, clauses for common table expressions * feat: add time-out and max-steps options for AST-based-reducer * feat: add support for simplifying double constant * feat: simplify double constants when using AST-based reducer * refactor: modify the structure of the code to make it appropriate * adjust the format of comments and code * change reduce-AST to reduce-ast --- docs/testCaseReduction.md | 4 +- src/sqlancer/ASTBasedReducer.java | 527 +++--------------- src/sqlancer/Main.java | 10 + src/sqlancer/MainOptions.java | 24 +- .../JSQLParserBasedTransformation.java | 36 ++ .../RemoveClausesOfSelect.java | 102 ++++ .../RemoveColumnsOfSelect.java | 64 +++ .../RemoveElementsOfExpressionList.java | 87 +++ .../transformations/RemoveRowsOfInsert.java | 45 ++ .../transformations/RemoveUnions.java | 52 ++ .../transformations/RoundDoubleConstant.java | 74 +++ .../transformations/SimplifyConstant.java | 103 ++++ .../transformations/SimplifyExpressions.java | 98 ++++ .../transformations/Transformation.java | 122 ++++ .../sqlancer/reducer/TestASTBasedReducer.java | 194 ++++++- 15 files changed, 1054 insertions(+), 488 deletions(-) create mode 100644 src/sqlancer/transformations/JSQLParserBasedTransformation.java create mode 100644 src/sqlancer/transformations/RemoveClausesOfSelect.java create mode 100644 src/sqlancer/transformations/RemoveColumnsOfSelect.java create mode 100644 src/sqlancer/transformations/RemoveElementsOfExpressionList.java create mode 100644 src/sqlancer/transformations/RemoveRowsOfInsert.java create mode 100644 src/sqlancer/transformations/RemoveUnions.java create mode 100644 src/sqlancer/transformations/RoundDoubleConstant.java create mode 100644 src/sqlancer/transformations/SimplifyConstant.java create mode 100644 src/sqlancer/transformations/SimplifyExpressions.java create mode 100644 src/sqlancer/transformations/Transformation.java diff --git a/docs/testCaseReduction.md b/docs/testCaseReduction.md index 35aa9cd08..c95770688 100644 --- a/docs/testCaseReduction.md +++ b/docs/testCaseReduction.md @@ -12,9 +12,9 @@ The AST-based reducer can shorten a statement by applying AST level transformati The transformations are implemented by [JSQLParser](https://github.com/JSQLParser/JSqlParser), a RDBMS agnostic SQL statement parser that can translate SQL statements into a traversable hierarchy of Java classes. JSQLParser provides support for the SQL standard as well as major SQL dialects. The AST-based reducer works for any SQL dialects that can be parsed by this tool. ## Enable reducers -Test-case reduction is disabled by default. The statement reducer can be enabled by passing `--use-reducer` when starting SQLancer. If you wish to further shorten each statements, you need to additionally pass the `--reduce-AST` parameter so that the AST-based reduction is applied. +Test-case reduction is disabled by default. The statement reducer can be enabled by passing `--use-reducer` when starting SQLancer. If you wish to further shorten each statements, you need to additionally pass the `--reduce-ast` parameter so that the AST-based reduction is applied. -Note: if `--reduce-AST` is set, `--use-reducer` option must be enabled first. +Note: if `--reduce-ast` is set, `--use-reducer` option must be enabled first. There are also options to define timeout seconds and max steps of reduction for both statement reducer and AST-based reducer. diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java index 705d9d307..36f16c2ac 100644 --- a/src/sqlancer/ASTBasedReducer.java +++ b/src/sqlancer/ASTBasedReducer.java @@ -1,101 +1,22 @@ package sqlancer; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.List; -import java.util.function.BiConsumer; -import net.sf.jsqlparser.expression.BinaryExpression; -import net.sf.jsqlparser.expression.CaseExpression; -import net.sf.jsqlparser.expression.DoubleValue; -import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.ExpressionVisitorAdapter; -import net.sf.jsqlparser.expression.LongValue; -import net.sf.jsqlparser.expression.NullValue; -import net.sf.jsqlparser.expression.Parenthesis; -import net.sf.jsqlparser.expression.StringValue; -import net.sf.jsqlparser.expression.WhenClause; -import net.sf.jsqlparser.expression.operators.relational.Between; -import net.sf.jsqlparser.expression.operators.relational.ExpressionList; -import net.sf.jsqlparser.expression.operators.relational.InExpression; -import net.sf.jsqlparser.expression.operators.relational.ItemsList; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.Statement; -import net.sf.jsqlparser.statement.StatementVisitorAdapter; -import net.sf.jsqlparser.statement.select.GroupByElement; -import net.sf.jsqlparser.statement.select.PlainSelect; -import net.sf.jsqlparser.statement.select.Select; -import net.sf.jsqlparser.statement.select.SelectBody; -import net.sf.jsqlparser.statement.select.SelectExpressionItem; -import net.sf.jsqlparser.statement.select.SelectVisitorAdapter; -import net.sf.jsqlparser.statement.select.SetOperationList; -import net.sf.jsqlparser.statement.select.SubSelect; -import net.sf.jsqlparser.statement.select.WithItem; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.transformations.RemoveClausesOfSelect; +import sqlancer.transformations.RemoveColumnsOfSelect; +import sqlancer.transformations.RemoveElementsOfExpressionList; +import sqlancer.transformations.RemoveRowsOfInsert; +import sqlancer.transformations.RemoveUnions; +import sqlancer.transformations.RoundDoubleConstant; +import sqlancer.transformations.SimplifyConstant; +import sqlancer.transformations.SimplifyExpressions; +import sqlancer.transformations.Transformation; -final class ExpressionTransformer { - - private static List flattenChildren(BinaryExpression expr) { - List candidates = new ArrayList<>(); - Expression lhs = expr.getLeftExpression(); - Expression rhs = expr.getRightExpression(); - candidates.add(lhs); - candidates.add(rhs); - return candidates; - } - - private static List flattenChildren(Between expr) { - Expression lhs = expr.getBetweenExpressionStart(); - Expression rhs = expr.getBetweenExpressionEnd(); - return List.of(lhs, rhs); - } - - public static List candidateExpressions(Expression expr) { - if (expr instanceof Parenthesis) { - // try removing a pair of brackets. - Parenthesis paren = (Parenthesis) expr; - return List.of(paren.getExpression()); - } else if (expr instanceof BinaryExpression) { - return flattenChildren((BinaryExpression) expr); - } else if (expr instanceof Between) { - return flattenChildren((Between) expr); - } else if (expr instanceof LongValue) { - LongValue longValue = (LongValue) expr; - if (String.valueOf(longValue).length() >= 4) { - return List.of(new NullValue(), new LongValue(10), new LongValue(0), new LongValue(1)); - } - return new ArrayList<>(); - } else if (expr instanceof DoubleValue) { - DoubleValue doubleValue = (DoubleValue) expr; - double literal = doubleValue.getValue(); - if (String.valueOf(literal).length() <= 4) { - return new ArrayList<>(); - } - double roundedValue = Math.round(literal * 10.0) / 10.0; - return List.of(new NullValue(), new DoubleValue(String.valueOf(roundedValue))); - } else if (expr instanceof StringValue) { - StringValue sv = (StringValue) expr; - String str = sv.getValue(); - if (str.length() > 4) { - return List.of(new NullValue(), new StringValue(" ")); - } - return new ArrayList<>(); - } else if (expr instanceof CaseExpression) { - CaseExpression caseExpression = (CaseExpression) expr; - return List.of(caseExpression.getSwitchExpression(), caseExpression.getElseExpression()); - } else { - return new ArrayList<>(); - } - } - - private ExpressionTransformer() throws Exception { - throw new AssertionError("Do not initialize the util class"); - } -} - -@SuppressWarnings("unchecked") public class ASTBasedReducer, O extends DBMSSpecificOptions, C extends SQLancerDBConnection> implements Reducer { @@ -105,406 +26,97 @@ public class ASTBasedReducer, O extends DBMSSpeci private G state; private G newGlobalState; private Reproducer reproducer; - private int reduceTargetIndex; - private Statement targetStatement; - // statement after reduction. private List> reducedStatements; + // statement after reduction. public ASTBasedReducer(DatabaseProvider provider) { this.provider = provider; } - private void updateStatements() { - String queryString = targetStatement.toString(); + @SuppressWarnings("unchecked") + private void updateStatements(String queryString, int index) { boolean couldAffectSchema = queryString.contains("CREATE TABLE") || queryString.contains("EXPLAIN"); - reducedStatements.set(reduceTargetIndex, (Query) new SQLQueryAdapter(queryString, couldAffectSchema)); - } - - public

void expressionReduce(P parent, Expression subExpr, // NOPMD - BiConsumer setter) { - boolean observeChange; - do { - observeChange = false; - List candidates = ExpressionTransformer.candidateExpressions(subExpr); - for (Expression candidate : candidates) { - try { - setter.accept(parent, candidate); - if (bugStillTriggers()) { - subExpr = candidate; - observeChange = true; - } - } catch (Exception ignoredException) { - } - } - setter.accept(parent, subExpr); - } while (observeChange); + reducedStatements.set(index, (Query) new SQLQueryAdapter(queryString, couldAffectSchema)); } - public void listElementRemovingReduce(P parent, List elms, // NOPMD - BiConsumer> setter) { - // TODO: For AST-Reducer, is delta-debugging needed ? Or just use the naive approach ? - boolean observeChange; - do { - observeChange = false; - for (int i = elms.size() - 1; i >= 0; i--) { - List reducedElms = new ArrayList<>(elms); - reducedElms.subList(i, i + 1).clear(); - setter.accept(parent, reducedElms); - try { - if (bugStillTriggers()) { - elms = reducedElms; - observeChange = true; - } - } catch (Exception e) { - System.out.println("An error occurred when trying executing reduced statements"); - e.printStackTrace(); - } - } - setter.accept(parent, elms); - } while (observeChange); - - } - - ExpressionVisitorAdapter expressionReducerVisitor = new ExpressionVisitorAdapter() { - - @Override - public void visit(InExpression expr) { - Expression rhs = expr.getRightExpression(); - - if (rhs instanceof SubSelect) { - SubSelect subSelect = (SubSelect) rhs; - subSelect.getSelectBody().accept(selectReducerVisitor); - } else { - ItemsList itemslist = expr.getRightItemsList(); - itemslist.accept(this); - } - } - - @Override - protected void visitBinaryExpression(BinaryExpression expr) { - Expression lhs = expr.getLeftExpression(); - Expression rhs = expr.getRightExpression(); - expressionReduce(expr, lhs, (expression, candidate) -> { - expression.setLeftExpression(candidate); - updateStatements(); - }); - expressionReduce(expr, rhs, (expression, candidate) -> { - expression.setRightExpression(candidate); - updateStatements(); - }); - lhs.accept(this); - rhs.accept(this); - } - - // @Override - // public void visit(DateValue value) { - // super.visit(value); - // } - // - // @Override - // public void visit(TimeValue value) { - // super.visit(value); - // } - // - // @Override - // public void visit(LikeExpression expr) { - // super.visit(expr); - // } - - @Override - public void visit(CaseExpression expr) { - Expression switchExpr = expr.getSwitchExpression(); - Expression elseExpr = expr.getElseExpression(); - - expressionReduce(expr, switchExpr, (parent, sw) -> { - parent.setSwitchExpression(sw); - updateStatements(); - }); - expressionReduce(expr, elseExpr, (parent, els) -> { - parent.setElseExpression(els); - updateStatements(); - }); - - super.visit(expr); - } - - @Override - public void visit(WhenClause whenClause) { - Expression when = whenClause.getWhenExpression(); - Expression then = whenClause.getThenExpression(); - - expressionReduce(whenClause, when, (wc, w) -> { - wc.setWhenExpression(w); - updateStatements(); - }); + @SuppressWarnings("unchecked") + @Override + public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { + this.state = state; + this.newGlobalState = newGlobalState; + this.reproducer = reproducer; - expressionReduce(whenClause, then, (wc, t) -> { - wc.setThenExpression(t); - updateStatements(); - }); + long maxReduceTime = state.getOptions().getMaxStatementReduceTime(); + long maxReduceSteps = state.getOptions().getMaxStatementReduceSteps(); - super.visit(whenClause); - } + List> initialBugInducingStatements = state.getState().getStatements(); + newGlobalState.getState().setStatements(new ArrayList<>(initialBugInducingStatements)); - @Override - public void visit(Parenthesis parenthesis) { - Expression closedExpr = parenthesis.getExpression(); - expressionReduce(parenthesis, closedExpr, (p, s) -> { - p.setExpression(s); - updateStatements(); - }); - closedExpr.accept(this); - } + List transformations = new ArrayList<>(); - // @Override - // public void visit(Function function) { - // super.visit(function); - // } + transformations.add(new RemoveUnions()); + transformations.add(new RemoveClausesOfSelect()); + transformations.add(new RemoveRowsOfInsert()); + transformations.add(new RemoveColumnsOfSelect()); + transformations.add(new RemoveElementsOfExpressionList()); + transformations.add(new SimplifyExpressions()); + transformations.add(new SimplifyConstant()); + transformations.add(new RoundDoubleConstant()); - @Override - public void visit(ExpressionList expressionList) { - List expressions = expressionList.getExpressions(); - listElementRemovingReduce(expressionList, expressions, (l, es) -> { - l.setExpressions(es); - updateStatements(); - }); - expressions = expressionList.getExpressions(); - for (int i = 0; i < expressions.size(); i++) { - Expression expr = expressions.get(i); - int index = i; - expressionReduce(expressions, expr, (l, e) -> { - l.set(index, e); - updateStatements(); - }); + Transformation.setBugJudgement(() -> { + try { + return this.bugStillTriggers(); + } catch (Exception ignored) { } - super.visit(expressionList); - } - - @Override - public void visit(SelectExpressionItem selectExpressionItem) { - Expression expr = selectExpressionItem.getExpression(); - expressionReduce(selectExpressionItem, expr, (item, e) -> { - item.setExpression(e); - updateStatements(); - }); - super.visit(selectExpressionItem); - } - - @Override - public void visit(SubSelect subSelect) { - subSelect.getSelectBody().accept(selectReducerVisitor); - } - - }; - - SelectVisitorAdapter selectReducerVisitor = new SelectVisitorAdapter() { - - // Clauses that would be tried removing. - // examples: - // Remove when: select * from table when 1 -> select * from table - // Remove limit: select * from table limit 1 -> select * from table - private final String[] removeList = { "Limit", "Offset", "Where", "Having", "GroupBy", "Distinct", - "OrderByElements", "Joins" }; + return false; + }); - // Clauses that would be tried transforming. - // examples: - // select * from t where a + b < c + d - // where clause might become one of the statement below after transformation: - // -> select * from table where c + d - // -> select * from table where a + b - - private final String[] transformList = { "Where", "Having", "FromItem", "SelectItems", "GroupBy", "Joins" }; - - // Clauses that would be visited for further reduction. - // example: - // select * from t where a + b < c + d - // Assuming that the coexistence of a and c would trigger the bug. - // The where clause : a + b < c + d would be visited and a + b, c + d would be reduced respectively. - // a + b < c + d might become a + c - private final String[] descendList = { "Where", "Having", "FromItem", "SelectItems", "GroupBy" }; - - private String getterName(String astNodeName) { - return "get" + astNodeName; - } - - private String setterName(String astNodeName) { - if (astNodeName.equals("GroupBy")) { - return "set" + astNodeName + "Element"; - } else { - return "set" + astNodeName; - } + boolean observeChange; + reducedStatements = new ArrayList<>(); + for (Query query : initialBugInducingStatements) { + reducedStatements.add((Query) query); } - @Override - public void visit(WithItem withItem) { - // withItem.getItemsList(); - } + Instant startTime = Instant.now(); + reduceProcess: do { + observeChange = false; + for (Transformation t : transformations) { + for (int i = 0; i < reducedStatements.size(); i++) { - @Override - public void visit(PlainSelect plainSelect) { - // transform section. Lists defined above would be iterated to get the corresponding clause name. Reflection - // is used to avoid repetitive code. e.g. The current astNodeName is When `getWhen`, `setWhen` would be - // called. - for (String astNodeName : removeList) { - try { - Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); - Object astNode = nodeGetter.invoke(plainSelect); - if (astNode == null) { - continue; + Instant currentTime = Instant.now(); + if (maxReduceTime != MainOptions.NO_REDUCE_LIMIT + && Duration.between(startTime, currentTime).getSeconds() >= maxReduceTime) { + break reduceProcess; } - Method nodeSetter = plainSelect.getClass().getMethod(setterName(astNodeName), - nodeGetter.getReturnType()); - nodeSetter.invoke(plainSelect, new Object[] { null }); - updateStatements(); - if (!bugStillTriggers()) { - nodeSetter.invoke(plainSelect, astNode); - updateStatements(); - } - } catch (Exception e) { - throw new AssertionError(e); - } - } - // Pull Up Section - for (String astNodeName : transformList) { - try { - Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); - Object astNode = nodeGetter.invoke(plainSelect); - if (astNode == null) { - continue; + if (maxReduceSteps != MainOptions.NO_REDUCE_LIMIT + && Transformation.getReduceSteps() >= maxReduceSteps) { + break reduceProcess; } - Method nodeSetter = plainSelect.getClass().getMethod(setterName(astNodeName), - nodeGetter.getReturnType()); - if (astNode instanceof Expression) { - Expression expr = (Expression) astNode; - expressionReduce(plainSelect, expr, (select, expression) -> { - try { - nodeSetter.invoke(select, expression); - updateStatements(); - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - } - }); - } else if (astNode instanceof List) { - List elms = (List) astNode; - if (elms.size() <= 1) { - continue; - } - listElementRemovingReduce(plainSelect, elms, (select, items) -> { - try { - nodeSetter.invoke(select, items); - updateStatements(); - } catch (IllegalAccessException | InvocationTargetException e) { - e.printStackTrace(); - } - }); - } else if (astNode instanceof GroupByElement) { - GroupByElement groupByElement = (GroupByElement) astNode; - ExpressionList expressionList = groupByElement.getGroupByExpressionList(); - if (expressionList == null) { - groupByElement.getGroupingSets(); - // TODO: TO BE IMPLEMENTED. - } else { - List elms = expressionList.getExpressions(); - if (elms.size() <= 1) { - continue; - } - listElementRemovingReduce(groupByElement, elms, (select, items) -> { - groupByElement.setGroupByExpressionList(new ExpressionList(items)); - updateStatements(); - }); - } - } - } catch (Exception e) { - throw new AssertionError(e); - } - } + Query query = reducedStatements.get(i); + boolean initFlag = t.init(query.getQueryString()); + int index = i; + t.setStatementChangedCallBack((statementString) -> { + updateStatements(statementString, index); + }); - for (String astNodeName : descendList) { - try { - Method nodeGetter = plainSelect.getClass().getMethod(getterName(astNodeName)); - Object astNode = nodeGetter.invoke(plainSelect); - if (astNode == null) { + if (!initFlag) { + System.out.println("Error when parsing the statement at transformer :" + t); continue; } - if (astNode instanceof Expression) { - ((Expression) astNode).accept(expressionReducerVisitor); - } else if (astNode instanceof List) { - // Really hacky... Some other ways to simplify it ? - List elms = (List) astNode; - for (Object obj : elms) { - if (obj instanceof SelectExpressionItem) { - ((SelectExpressionItem) obj).accept(expressionReducerVisitor); - } else if (obj instanceof Expression) { - ((Expression) obj).accept(expressionReducerVisitor); - } - } - } else if (astNode instanceof GroupByElement) { - GroupByElement groupByElement = (GroupByElement) astNode; - ExpressionList expressionList = groupByElement.getGroupByExpressionList(); - if (expressionList != null) { - expressionList.accept(expressionReducerVisitor); - } - // TODO: groupByElement.getGroupingSets() TO BE IMPLEMENTED - - } - } catch (Exception e) { - throw new AssertionError(e); - } - } - } - - @Override - public void visit(SetOperationList setOpList) { - List selectBodies = setOpList.getSelects(); - listElementRemovingReduce(setOpList, selectBodies, (optionList, selects) -> { - optionList.setSelects(selects); - updateStatements(); - }); - for (SelectBody selectBody : selectBodies) { - if (selectBody instanceof PlainSelect) { - visit((PlainSelect) selectBody); + t.apply(); + observeChange |= t.changed(); } } - } - - }; - - StatementVisitorAdapter statementReducerVisitor = new StatementVisitorAdapter() { - @Override - public void visit(Select select) { - SelectBody selectBody = select.getSelectBody(); - if (selectBody != null) { - selectBody.accept(selectReducerVisitor); - } - } - }; - - @Override - public void reduce(G state, Reproducer reproducer, G newGlobalState) throws Exception { - this.state = state; - this.newGlobalState = newGlobalState; - this.reproducer = reproducer; - - List> initialBugInducingStatements = state.getState().getStatements(); - reducedStatements = new ArrayList<>(); - for (Query query : initialBugInducingStatements) { - reducedStatements.add((Query) query); - } - - for (int i = 0; i < reducedStatements.size(); i++) { - reduceTargetIndex = i; - Query query = reducedStatements.get(reduceTargetIndex); - targetStatement = CCJSqlParserUtil.parse(query.getQueryString()); - targetStatement.accept(statementReducerVisitor); - } + } while (observeChange); newGlobalState.getState().setStatements(new ArrayList<>(reducedStatements)); + newGlobalState.getLogger().logReduced(newGlobalState.getState()); } - private boolean bugStillTriggers() throws Exception { + public boolean bugStillTriggers() throws Exception { try (C con2 = provider.createDatabase(newGlobalState)) { newGlobalState.setConnection(con2); List> candidateStatements = new ArrayList<>(reducedStatements); @@ -519,6 +131,7 @@ private boolean bugStillTriggers() throws Exception { } try { if (reproducer.bugStillTriggers(newGlobalState)) { + newGlobalState.getLogger().logReduced(newGlobalState.getState()); return true; } } catch (Throwable ignoredException) { diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 70657c3d6..4b480e595 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -444,6 +444,10 @@ public void run() throws Exception { } catch (IOException e) { throw new AssertionError(e); } + + if (options.reduceAST() && !options.useReducer()) { + throw new AssertionError("To reduce AST, use-reducer option must be enabled first"); + } if (reproducer != null && options.useReducer()) { System.out.println("EXPERIMENTAL: Trying to reduce queries using a simple reducer."); // System.out.println("Reduced query will be output to stdout but not logs."); @@ -459,6 +463,12 @@ public void run() throws Exception { Reducer reducer = new StatementReducer<>(provider); reducer.reduce(state, reproducer, newGlobalState); + + if (options.reduceAST()) { + Reducer astBasedReducer = new ASTBasedReducer<>(provider); + astBasedReducer.reduce(state, reproducer, newGlobalState); + } + throw new AssertionError("Found a potential bug"); } } diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index c653a358f..232856f94 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -126,10 +126,19 @@ public class MainOptions { @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") private boolean useReducer = false; // NOPMD + @Parameter(names = "--reduce-ast", description = "EXPERIMENTAL perform AST reduction after statement reduction") + private boolean reduceAST = false; // NOPMD + @Parameter(names = "--statement-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the statement reducer will do") private long maxStatementReduceSteps = NO_REDUCE_LIMIT; // NOPMD - @Parameter(names = "--statement-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the statement reducer will do") + @Parameter(names = "--statement-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the AST-based reducer will do") + private long maxASTReduceTime = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--ast-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the AST-based reducer will do") + private long maxASTReduceSteps = NO_REDUCE_LIMIT; // NOPMD + + @Parameter(names = "--ast-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the statement reducer will do") private long maxStatementReduceTime = NO_REDUCE_LIMIT; // NOPMD public int getMaxExpressionDepth() { @@ -293,6 +302,10 @@ public boolean useReducer() { return useReducer; } + public boolean reduceAST() { + return reduceAST; + } + public long getMaxStatementReduceSteps() { return maxStatementReduceSteps; } @@ -300,4 +313,13 @@ public long getMaxStatementReduceSteps() { public long getMaxStatementReduceTime() { return maxStatementReduceTime; } + + public long getMaxASTReduceSteps() { + return maxASTReduceSteps; + } + + public long getMaxASTReduceTime() { + return maxASTReduceTime; + } + } diff --git a/src/sqlancer/transformations/JSQLParserBasedTransformation.java b/src/sqlancer/transformations/JSQLParserBasedTransformation.java new file mode 100644 index 000000000..e7b7b6145 --- /dev/null +++ b/src/sqlancer/transformations/JSQLParserBasedTransformation.java @@ -0,0 +1,36 @@ +package sqlancer.transformations; + +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.Statement; + +/** + * Transformations based on JSQLParser should be derived from this class. + */ + +public class JSQLParserBasedTransformation extends Transformation { + + protected Statement statement; + + public JSQLParserBasedTransformation(String desc) { + super(desc); + } + + @Override + protected void onStatementChanged() { + if (statementChangedHandler != null) { + statementChangedHandler.accept(this.statement.toString()); + } + } + + @Override + public boolean init(String sql) { + this.current = sql; + try { + statement = CCJSqlParserUtil.parse(current); + } catch (Exception e) { + return false; + } + return true; + } + +} diff --git a/src/sqlancer/transformations/RemoveClausesOfSelect.java b/src/sqlancer/transformations/RemoveClausesOfSelect.java new file mode 100644 index 000000000..832f82762 --- /dev/null +++ b/src/sqlancer/transformations/RemoveClausesOfSelect.java @@ -0,0 +1,102 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.statement.select.Distinct; +import net.sf.jsqlparser.statement.select.GroupByElement; +import net.sf.jsqlparser.statement.select.Limit; +import net.sf.jsqlparser.statement.select.Offset; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SubSelect; +import net.sf.jsqlparser.statement.select.WithItem; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * remove clauses of a select, such as join, where, group by, distinct, offset, limit. + * + * e.g. select * from t where a = b offset 1 limit 1 -> select * from t; + */ + +public class RemoveClausesOfSelect extends JSQLParserBasedTransformation { + private final SelectDeParser remover = new SelectDeParser() { + @Override + public void visit(PlainSelect plainSelect) { + handleSelect(plainSelect); + super.visit(plainSelect); + } + }; + + public RemoveClausesOfSelect() { + super("remove clauses of select"); + } + + @Override + public boolean init(String original) { + + boolean baseSuc = super.init(original); + if (!baseSuc) { + return false; + } + + this.remover.setExpressionVisitor(new ExpressionDeParser(remover, new StringBuilder())); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(remover); + + List withItemsList = select.getWithItemsList(); + if (withItemsList == null) { + return; + } + tryRemoveElms(select, withItemsList, Select::setWithItemsList); + + for (WithItem withItem : withItemsList) { + SubSelect subSelect = withItem.getSubSelect(); + if (subSelect == null) { + return; + } + + if (subSelect.getSelectBody() != null) { + subSelect.getSelectBody().accept(remover); + } + } + } + } + + private void handleSelect(PlainSelect plainSelect) { + + Expression where = plainSelect.getWhere(); + if (where != null) { + tryRemove(plainSelect, where, PlainSelect::setWhere); + } + + GroupByElement groupByElement = plainSelect.getGroupBy(); + if (groupByElement != null) { + tryRemove(plainSelect, groupByElement, PlainSelect::setGroupByElement); + } + + Distinct distinct = plainSelect.getDistinct(); + if (distinct != null) { + tryRemove(plainSelect, distinct, PlainSelect::setDistinct); + } + + Offset offset = plainSelect.getOffset(); + if (offset != null) { + tryRemove(plainSelect, offset, PlainSelect::setOffset); + } + + Limit limit = plainSelect.getLimit(); + if (offset != null) { + tryRemove(plainSelect, limit, PlainSelect::setLimit); + } + } + +} diff --git a/src/sqlancer/transformations/RemoveColumnsOfSelect.java b/src/sqlancer/transformations/RemoveColumnsOfSelect.java new file mode 100644 index 000000000..6cdd44d15 --- /dev/null +++ b/src/sqlancer/transformations/RemoveColumnsOfSelect.java @@ -0,0 +1,64 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SubSelect; +import net.sf.jsqlparser.statement.select.WithItem; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * remove columns of a select: e.g. select a, b, c from t -> select a from t. + */ +public class RemoveColumnsOfSelect extends JSQLParserBasedTransformation { + + private final SelectDeParser remover = new SelectDeParser() { + @Override + public void visit(PlainSelect plainSelect) { + tryRemoveElms(plainSelect, plainSelect.getSelectItems(), PlainSelect::setSelectItems); + super.visit(plainSelect); + } + }; + + public RemoveColumnsOfSelect() { + super("remove columns of a select"); + } + + @Override + public boolean init(String original) { + + boolean baseSucc = super.init(original); + if (!baseSucc) { + return false; + } + this.remover.setExpressionVisitor(new ExpressionDeParser(remover, new StringBuilder())); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(remover); + + List withItemsList = select.getWithItemsList(); + if (withItemsList == null) { + return; + } + for (WithItem withItem : withItemsList) { + SubSelect subSelect = withItem.getSubSelect(); + if (subSelect == null) { + return; + } + + if (subSelect.getSelectBody() != null) { + subSelect.getSelectBody().accept(remover); + } + } + + } + } +} diff --git a/src/sqlancer/transformations/RemoveElementsOfExpressionList.java b/src/sqlancer/transformations/RemoveElementsOfExpressionList.java new file mode 100644 index 000000000..ebeb0b0af --- /dev/null +++ b/src/sqlancer/transformations/RemoveElementsOfExpressionList.java @@ -0,0 +1,87 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.statement.select.GroupByElement; +import net.sf.jsqlparser.statement.select.Join; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.InsertDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * remove elements of an expression list. + * + * NOTE: this only works for select statements and targets at ExpressionList type in JSQLParser, such as groupBy list + */ +public class RemoveElementsOfExpressionList extends JSQLParserBasedTransformation { + private final ExpressionDeParser expressionHandler = new ExpressionDeParser(); + private final SelectDeParser simplifier = new SelectDeParser() { + @Override + public void visit(PlainSelect plainSelect) { + handleSelect(plainSelect); + super.visit(plainSelect); + } + + @Override + public void visit(ExpressionList expressionList) { + List expressions = expressionList.getExpressions(); + tryRemoveElms(expressionList, expressions, ExpressionList::setExpressions); + super.visit(expressionList); + } + }; + private final InsertDeParser insertDeParser = new InsertDeParser() { + @Override + public void visit(ExpressionList expressionList) { + List expressions = expressionList.getExpressions(); + tryRemoveElms(expressionList, expressions, ExpressionList::setExpressions); + super.visit(expressionList); + } + }; + + public RemoveElementsOfExpressionList() { + super("remove elements of expression lists"); + } + + @Override + public boolean init(String sql) { + boolean baseSuc = super.init(sql); + if (!baseSuc) { + return false; + } + this.simplifier.setExpressionVisitor(expressionHandler); + this.expressionHandler.setSelectVisitor(simplifier); + + this.insertDeParser.setExpressionVisitor(expressionHandler); + this.insertDeParser.setSelectVisitor(simplifier); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(simplifier); + } + } + + private void handleSelect(PlainSelect plainSelect) { + + GroupByElement groupByElement = plainSelect.getGroupBy(); + + if (groupByElement != null && groupByElement.getGroupByExpressionList() != null) { + ExpressionList expressionList = groupByElement.getGroupByExpressionList(); + List list = expressionList.getExpressions(); + tryRemoveElms(expressionList, list, ExpressionList::setExpressions); + } + + List expressionList = plainSelect.getJoins(); + if (expressionList != null) { + tryRemoveElms(plainSelect, expressionList, PlainSelect::setJoins); + } + } +} diff --git a/src/sqlancer/transformations/RemoveRowsOfInsert.java b/src/sqlancer/transformations/RemoveRowsOfInsert.java new file mode 100644 index 000000000..b49f80149 --- /dev/null +++ b/src/sqlancer/transformations/RemoveRowsOfInsert.java @@ -0,0 +1,45 @@ +package sqlancer.transformations; + +import net.sf.jsqlparser.expression.operators.relational.ExpressionList; +import net.sf.jsqlparser.expression.operators.relational.ItemsList; +import net.sf.jsqlparser.statement.insert.Insert; +import net.sf.jsqlparser.statement.select.SelectBody; +import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.statement.values.ValuesStatement; + +/** + * This Transformer remove rows of insert. Given a sql statement: + * + * INSERT INTO t1(c2, c0) VALUES (1508438260, 2929), (1508438260, TIMESTAMP '1969-12-26 01:57:21'), (0.5347171705591047, + * 398662142); -> INSERT INTO t1 (c2, c0) VALUES (0.5347171705591047, 398662142); + */ +public class RemoveRowsOfInsert extends JSQLParserBasedTransformation { + public RemoveRowsOfInsert() { + super("remove rows of an insert statement"); + } + + @Override + public void apply() { + super.apply(); + if (!(statement instanceof Insert)) { + return; + } + SelectBody selectBody = ((Insert) statement).getSelect().getSelectBody(); + if (!(selectBody instanceof SetOperationList)) { + return; + } + SetOperationList insertingList = (SetOperationList) selectBody; + for (SelectBody selBody : insertingList.getSelects()) { + if (!(selBody instanceof ValuesStatement)) { + continue; + } + ValuesStatement valuesStatement = (ValuesStatement) selBody; + ItemsList itemsList = valuesStatement.getExpressions(); + if (!(itemsList instanceof ExpressionList)) { + continue; + } + tryRemoveElms((ExpressionList) itemsList, ((ExpressionList) itemsList).getExpressions(), + ExpressionList::setExpressions); + } + } +} diff --git a/src/sqlancer/transformations/RemoveUnions.java b/src/sqlancer/transformations/RemoveUnions.java new file mode 100644 index 000000000..4f9593345 --- /dev/null +++ b/src/sqlancer/transformations/RemoveUnions.java @@ -0,0 +1,52 @@ +package sqlancer.transformations; + +import java.util.List; + +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.statement.select.SelectBody; +import net.sf.jsqlparser.statement.select.SetOperationList; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * try removing sub selects of a union statement. + * + * e.g. select 1 union select 2 -> select 1 + */ + +public class RemoveUnions extends JSQLParserBasedTransformation { + + private final SelectDeParser remover = new SelectDeParser() { + @Override + public void visit(SetOperationList list) { + List selectBodyList = list.getSelects(); + tryRemoveElms(list, selectBodyList, SetOperationList::setSelects); + super.visit(list); + } + }; + + public RemoveUnions() { + super("remove union selects"); + } + + @Override + public boolean init(String sql) { + + boolean baseSuc = super.init(sql); + if (!baseSuc) { + return false; + } + + this.remover.setExpressionVisitor(new ExpressionDeParser(remover, new StringBuilder())); + return true; + } + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(remover); + } + } +} diff --git a/src/sqlancer/transformations/RoundDoubleConstant.java b/src/sqlancer/transformations/RoundDoubleConstant.java new file mode 100644 index 000000000..d22496c4e --- /dev/null +++ b/src/sqlancer/transformations/RoundDoubleConstant.java @@ -0,0 +1,74 @@ +package sqlancer.transformations; + +import java.text.DecimalFormat; +import java.util.HashSet; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Round double values which are longer than a certain length. e.g. 2.4782565267 -> 2.478. + * + * This transformation is not based on JSQLParser. + */ +public class RoundDoubleConstant extends Transformation { + + private Set doubleValueCollector; + + private String currentString; + + private static final int ROUND_LENGTH = 3; + private DecimalFormat decimalFormat; + + public RoundDoubleConstant() { + super("round double constant values"); + } + + @Override + public boolean init(String sql) { + super.init(sql); + decimalFormat = new DecimalFormat("#." + "#".repeat(ROUND_LENGTH)); + + currentString = sql; + doubleValueCollector = new HashSet<>(); + + String regex = "\\b-?\\d+\\.\\d+\\b"; + + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(sql); + + while (matcher.find()) { + String matchedText = matcher.group(); + String decimalPart = matchedText.replaceAll("\\d+\\.", ""); + int decimalPlaces = decimalPart.length(); + if (decimalPlaces > ROUND_LENGTH) { + doubleValueCollector.add(matchedText); + } + } + return true; + } + + @Override + public void apply() { + for (String doubleValue : doubleValueCollector) { + + double targetNumber = Double.parseDouble(doubleValue); + String roundedNumberStr = decimalFormat.format(targetNumber); + + String replacement = currentString.replace(doubleValue, roundedNumberStr); + String original = currentString; + + tryReplace(null, original, replacement, (p, r) -> { + currentString = r; + }); + } + super.apply(); + } + + @Override + protected void onStatementChanged() { + if (statementChangedHandler != null) { + statementChangedHandler.accept(currentString); + } + } +} diff --git a/src/sqlancer/transformations/SimplifyConstant.java b/src/sqlancer/transformations/SimplifyConstant.java new file mode 100644 index 000000000..6638dca73 --- /dev/null +++ b/src/sqlancer/transformations/SimplifyConstant.java @@ -0,0 +1,103 @@ +package sqlancer.transformations; + +import java.util.ArrayList; +import java.util.List; + +import net.sf.jsqlparser.expression.DoubleValue; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.LongValue; +import net.sf.jsqlparser.expression.StringValue; +import net.sf.jsqlparser.statement.StatementVisitorAdapter; +import net.sf.jsqlparser.statement.insert.Insert; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * Shorten the constant of a statement e.g. "a_very_long_str" -> "_", 12341234->1. + * + * Note: The API of JSQLParser may have some problems with double values: `setValue` can't change the literal value of a + * DoubleValue object. Therefore, double values are handled at RoundDoubleConstant class. + */ +public class SimplifyConstant extends JSQLParserBasedTransformation { + static class ConstantCollector extends ExpressionDeParser { + private final List candidates = new ArrayList<>(); + + @Override + public void visit(DoubleValue doubleValue) { + candidates.add(doubleValue); + super.visit(doubleValue); + } + + @Override + public void visit(LongValue longValue) { + candidates.add(longValue); + super.visit(longValue); + } + + @Override + public void visit(StringValue stringValue) { + candidates.add(stringValue); + super.visit(stringValue); + } + + public List getCandidates() { + return candidates; + } + } + + public SimplifyConstant() { + super("simplify constant expressions"); + } + + @Override + public void apply() { + super.apply(); + ConstantCollector collector = new ConstantCollector(); + StringBuilder buffer = new StringBuilder(); + SelectDeParser collectorDeParser = new SelectDeParser(collector, buffer); + collector.setSelectVisitor(collectorDeParser); + collector.setBuffer(buffer); + + List candidates = collector.getCandidates(); + + StatementVisitorAdapter statementVisitor = new StatementVisitorAdapter() { + @Override + public void visit(Insert insert) { + insert.getSelect().getSelectBody().accept(collectorDeParser); + super.visit(insert); + } + + @Override + public void visit(Select select) { + select.getSelectBody().accept(collectorDeParser); + super.visit(select); + } + }; + + statement.accept(statementVisitor); + + for (Expression e : candidates) { + if (e instanceof LongValue) { + simplify((LongValue) e); + } else if (e instanceof StringValue) { + simplify((StringValue) e); + } + } + } + + private void simplify(LongValue longValue) { + long variant = 0; + if (!longValue.getStringValue().equals(String.valueOf(variant))) { + tryReplace(longValue, longValue.getStringValue(), String.valueOf(variant), LongValue::setStringValue); + } + } + + private void simplify(StringValue stringValue) { + String variant = "_"; + if (!stringValue.getValue().equals(variant)) { + tryReplace(stringValue, stringValue.getValue(), variant, StringValue::setValue); + } + } + +} diff --git a/src/sqlancer/transformations/SimplifyExpressions.java b/src/sqlancer/transformations/SimplifyExpressions.java new file mode 100644 index 000000000..244629b4d --- /dev/null +++ b/src/sqlancer/transformations/SimplifyExpressions.java @@ -0,0 +1,98 @@ +package sqlancer.transformations; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; + +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.Parenthesis; +import net.sf.jsqlparser.statement.select.PlainSelect; +import net.sf.jsqlparser.statement.select.Select; +import net.sf.jsqlparser.util.deparser.ExpressionDeParser; +import net.sf.jsqlparser.util.deparser.SelectDeParser; + +/** + * This transformation simplifies complicated expressions e.g: a + (b + c) -> b. + */ + +public class SimplifyExpressions extends JSQLParserBasedTransformation { + public SimplifyExpressions() { + super("simplify expressions. e.g. a + b -> a"); + } + + @Override + public boolean init(String sql) { + boolean baseSuc = super.init(sql); + if (!baseSuc) { + return false; + } + this.simplifier.setExpressionVisitor(expressionHandler); + this.expressionHandler.setSelectVisitor(simplifier); + return true; + } + + private final ExpressionDeParser expressionHandler = new ExpressionDeParser() { + @Override + protected void visitBinaryExpression(BinaryExpression binaryExpression, String operator) { + + Expression lhs = binaryExpression.getLeftExpression(); + Expression rhs = binaryExpression.getRightExpression(); + + handleExpression(binaryExpression, lhs, BinaryExpression::setLeftExpression); + handleExpression(binaryExpression, rhs, BinaryExpression::setRightExpression); + + super.visitBinaryExpression(binaryExpression, operator); + } + + }; + private final SelectDeParser simplifier = new SelectDeParser() { + + @Override + public void visit(PlainSelect plainSelect) { + handleSelect(plainSelect); + super.visit(plainSelect); + } + }; + + @Override + public void apply() { + super.apply(); + if (statement instanceof Select) { + Select select = (Select) statement; + select.getSelectBody().accept(simplifier); + } + } + + private void handleSelect(PlainSelect plainSelect) { + Expression where = plainSelect.getWhere(); + if (where != null) { + handleExpression(plainSelect, where, PlainSelect::setWhere); + } + Expression having = plainSelect.getHaving(); + if (having != null) { + handleExpression(plainSelect, having, PlainSelect::setHaving); + } + } + + private List flattenExpression(Expression expression) { + if (expression instanceof BinaryExpression) { + BinaryExpression binaryExpression = (BinaryExpression) expression; + return List.of(binaryExpression.getLeftExpression(), binaryExpression.getRightExpression()); + } else if (expression instanceof Parenthesis) { + return List.of(((Parenthesis) expression).getExpression()); + } + return new ArrayList<>(); + } + + private

void handleExpression(P parent, Expression expr, BiConsumer setter) { + + List expressions = flattenExpression(expr); + for (Expression variant : expressions) { + boolean suc = tryReplace(parent, expr, variant, setter); + if (suc) { + break; + } + } + } +} diff --git a/src/sqlancer/transformations/Transformation.java b/src/sqlancer/transformations/Transformation.java new file mode 100644 index 000000000..affa4c281 --- /dev/null +++ b/src/sqlancer/transformations/Transformation.java @@ -0,0 +1,122 @@ +package sqlancer.transformations; + +import java.util.ArrayList; +import java.util.List; +import java.util.function.BiConsumer; +import java.util.function.Consumer; +import java.util.function.Supplier; + +/** + * The base class of transformations. Defines APIs to remove, replace, remove elements of a list. + */ +public class Transformation { + + private static Supplier bugJudgement; + private static long reduceSteps; + + protected boolean isChanged; + protected String current; + protected String desc = ""; + + protected Consumer statementChangedHandler; + + public Transformation(String desc) { + this.desc = desc; + } + + @SuppressWarnings("unused") + protected Transformation() { + } + + public static void setBugJudgement(Supplier judgement) { + bugJudgement = judgement; + } + + @Override + public String toString() { + return desc; + } + + public boolean init(String sql) { + isChanged = false; + return true; + } + + public boolean tryRemove(P parent, T target, BiConsumer setter) { + setter.accept(parent, null); + onStatementChanged(); + if (!bugStillTriggers()) { + setter.accept(parent, target); + onStatementChanged(); + return false; + } + reduceSteps++; + isChanged = true; + return true; + } + + public boolean tryReplace(P parent, T original, T vari, BiConsumer setter) { + setter.accept(parent, vari); + onStatementChanged(); + if (!bugStillTriggers()) { + setter.accept(parent, original); + onStatementChanged(); + return false; + } + reduceSteps++; + isChanged = true; + return true; + } + + public void tryRemoveElms(P parent, List elms, // NOPMD + BiConsumer> setter) { + boolean observeChange; + do { + observeChange = false; + for (int i = elms.size() - 1; i >= 0; i--) { + List reducedElms = new ArrayList<>(elms); + reducedElms.subList(i, i + 1).clear(); + setter.accept(parent, reducedElms); + onStatementChanged(); + if (bugStillTriggers()) { + elms = reducedElms; + onStatementChanged(); + observeChange = true; + } + } + isChanged |= observeChange; + setter.accept(parent, elms); + reduceSteps++; + onStatementChanged(); + } while (observeChange); + + } + + public boolean bugStillTriggers() { + try { + return Transformation.bugJudgement.get(); + } catch (Exception ignored) { + } + return false; + } + + public void apply() { + isChanged = false; + } + + public boolean changed() { + return isChanged; + } + + public static long getReduceSteps() { + return reduceSteps; + } + + protected void onStatementChanged() { + } + + public void setStatementChangedCallBack(Consumer statementChangedHandler) { + this.statementChangedHandler = statementChangedHandler; + } + +} diff --git a/test/sqlancer/reducer/TestASTBasedReducer.java b/test/sqlancer/reducer/TestASTBasedReducer.java index b65ceef21..549bbe84d 100644 --- a/test/sqlancer/reducer/TestASTBasedReducer.java +++ b/test/sqlancer/reducer/TestASTBasedReducer.java @@ -1,14 +1,19 @@ package sqlancer.reducer; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.junit.jupiter.api.Test; import sqlancer.common.query.Query; -import java.util.Arrays; import java.util.List; +import java.util.function.Function; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; public class TestASTBasedReducer { @Test - void testLongStatement() throws Exception { + void testRemovingClauses() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); String[] queriesStr = { @@ -16,12 +21,17 @@ void testLongStatement() throws Exception { env.setInitialStatementsFromStrings(List.of(queriesStr)); env.setBugInducingCondition(statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } return queriesString.contains("&&"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStr)); - System.out.println(reducedResult); + String outcome = TestEnvironment.getQueriesString(reducedResult); + assertEquals(outcome, "SELECT * FROM v0 WHERE v0.c && v0.c;"); } @Test @@ -33,12 +43,16 @@ void testReducingMultipleTokensToOne() throws Exception { env.setInitialStatementsFromStrings(List.of(queriesStr)); env.setBugInducingCondition(statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } return queriesString.contains("||"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStr)); - System.out.println(reducedResult); + assertEquals(TestEnvironment.getQueriesString(reducedResult), "SELECT row_id FROM v0 WHERE v0.rowid || 0;"); } @Test @@ -47,83 +61,207 @@ void testMultipleStatements() throws Exception { String[] queriesStrs = { "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)", - "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)", - "SELECT * FROM table_3;" }; + "SELECT DISTINCT row_id, c FROM v0 WHERE ((v0.rowid || (v0.c < 200 && v0.c >= 100) || 114514)OR(((v0.c0)||(1529686005)))) UNION SELECT DISTINCT * FROM v0 WHERE (NOT ((v0.rowid)OR(((v0.c0)||(1529686005))))) UNION SELECT DISTINCT * FROM v0 WHERE ((((v0.rowid)OR(((v0.c0)||(1529686005))))) IS NULL)" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); env.setBugInducingCondition(statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + return queriesString.toUpperCase().contains("UNION"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals(TestEnvironment.getQueriesString(reducedResult), + "SELECT row_id FROM v0;\nSELECT row_id FROM v0 UNION SELECT * FROM v0;"); } @Test - void testJoin() throws Exception { + void removeJoins() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); String[] queriesStrs = { "SELECT * FROM t0, t1, t2, t3, t4 Where t2.val = t1.val" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); env.setBugInducingCondition(statements -> { + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } String queriesString = TestEnvironment.getQueriesString(statements); - return queriesString.contains("t1") && queriesString.contains("WHERE"); + if (!queriesString.contains("WHERE")) { + return false; + } + String[] split = queriesString.split("WHERE"); + String columns = split[0]; + String condition = split[1]; + return columns.contains("t1") && condition.contains("t1"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals(TestEnvironment.getQueriesString(reducedResult), "SELECT * FROM t0, t1 WHERE t1.val;"); } @Test - void testCase() throws Exception { + void testComplicated() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); String[] queriesStrs = { "SELECT STRING_AGG(v0.c2) FROM t0, v0 WHERE (CASE true WHEN (ABS(true) BETWEEN (v0.c0 LIKE NULL ESCAPE v0.c2) AND (DATE '1970-01-23' NOT IN (v0.c2))) THEN (0.07914839711718646 NOT BETWEEN '' AND ((v0.c0)OR(v0.c2))) WHEN v0.c1 THEN ((v0.c1)-(v0.c0)) WHEN t0.c1 THEN (TIMESTAMP '1969-12-29 20:22:33' IN (PI(), v0.c2, (v0.c1 BETWEEN '' AND v0.rowid))) WHEN v0.c1 THEN TIMESTAMP '1969-12-16 17:24:43' WHEN ((((v0.c1)-(t0.c0)))||(t0.c0)) THEN true ELSE ((0.279978719843174)/(((v0.c1)>(DATE '1969-12-19')))) END ) GROUP BY ((DATE '1970-01-24') IS NULL), t0.c1, (CASE (v0.c1 LIKE ((0.9833120083624495)SIMILAR TO(t0.rowid)) ESCAPE CEIL(TIMESTAMP '1970-01-11 16:38:26')) WHEN t0.rowid THEN 0.27742217994251717 ELSE ((v0.c0) IS NOT NULL) END );" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); - env.setBugInducingCondition(statements -> { + Function>, Boolean> condition = statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } return queriesString.toUpperCase().contains("CASE"); + }; + env.setBugInducingCondition(condition); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertTrue(condition.apply(reducedResult)); + } + + @Test + void testSimplifyingConstantStringValue() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { "SELECT * FROM t0 WHERE v LIKE '[vQ3㭫oQ';" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + + String queriesString = TestEnvironment.getQueriesString(statements); + return queriesString.contains("LIKE"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals("SELECT * FROM t0 WHERE v LIKE '_';", TestEnvironment.getQueriesString(reducedResult)); } @Test - void testConstantVar() throws Exception { + void testSimplifyingConstantLongValue() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); - String[] queriesStrs = { - "SELECT STRING_AGG(v0.c2) FROM t0 GROUP BY ( (CASE (t0.rowid) WHEN t0.rowid THEN 0.27742217994251717 ELSE ((v0.c0) IS NOT NULL) END) )" }; + String[] queriesStrs = { "SELECT * FROM t0 where t0.v = 314598267;" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); env.setBugInducingCondition(statements -> { + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } + String queriesString = TestEnvironment.getQueriesString(statements); - return queriesString.toUpperCase().contains("CASE"); + return queriesString.contains("t0.v = "); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals("SELECT * FROM t0 WHERE t0.v = 0;", TestEnvironment.getQueriesString(reducedResult)); } @Test - void testFunction() throws Exception { + void testSubSelects() throws Exception { TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); String[] queriesStrs = { - "SELECT DATE '1970-01-11', false, t1.c1, t1.c1, (t1.c1 NOT IN (((('' LIKE t1.c2 ESCAPE t1.c1)) IS NOT NULL))) FROM t1 WHERE t1.c0 GROUP BY (((('Zlb)' IN (t1.c0)) LIKE t1.c1 ESCAPE (0.6419925594156123 BETWEEN t1.c1 AND ')-'))) ::BOOL) HAVING ((LAST_DAY(1630554083))&(AVG((CASE t1.c0 WHEN t1.c2 THEN DATE '1970-01-09' ELSE '' END )))) LIMIT 714775291;" }; + "SELECT AVG(c0) FROM (SELECT SUM(c1) AS c0 FROM t1 GROUP BY c2 LIMIT 32 OFFSET 128) AS t1;" }; env.setInitialStatementsFromStrings(List.of(queriesStrs)); env.setBugInducingCondition(statements -> { String queriesString = TestEnvironment.getQueriesString(statements); + try { + for (Query s : statements) { + CCJSqlParserUtil.parse(s.getQueryString()); + } + } catch (JSQLParserException e) { + return false; + } return queriesString.contains("AVG"); }); env.runReduce(); List> reducedResult = env.getReducedStatements(); - System.out.println(Arrays.toString(queriesStrs)); - System.out.println(reducedResult); + assertEquals("SELECT AVG(c0) FROM (SELECT SUM(c1) AS c0 FROM t1) AS t1;", + TestEnvironment.getQueriesString(reducedResult)); + } + + @Test + void testInsert() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "INSERT INTO t1(c2, c0) VALUES (1508438260, 2929), (1508438260, TIMESTAMP '1969-12-26 01:57:21'), (0.5347171705591047, 398662142);" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("(0.5347171705591047, 398662142)"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("INSERT INTO t1 (c2, c0) VALUES (0.5347171705591047, 398662142);", + TestEnvironment.getQueriesString(reducedResult)); } + + @Test + void testWithSelect() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + + String[] queriesStrs = { + "WITH cte1 AS (SELECT a, b FROM table1 where a < b), cte2 AS (SELECT c, d FROM table2 where c = d) SELECT b, d FROM cte1 JOIN cte2 WHERE cte1.a = cte2.c;" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("table1"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("WITH cte1 AS (SELECT a FROM table1) SELECT b FROM cte1;", + TestEnvironment.getQueriesString(reducedResult)); + } + + @Test + void testRoundDouble() throws Exception { + TestEnvironment env = TestEnvironment.getASTBasedReducerEnv(); + String[] queriesStrs = { "SELECT * FROM t0 WHERE (2.1427572639 IS NULL);" }; + env.setInitialStatementsFromStrings(List.of(queriesStrs)); + env.setBugInducingCondition(statements -> { + String queriesString = TestEnvironment.getQueriesString(statements); + try { + CCJSqlParserUtil.parse(queriesString); + } catch (JSQLParserException e) { + return false; + } + return queriesString.contains("WHERE"); + }); + env.runReduce(); + List> reducedResult = env.getReducedStatements(); + assertEquals("SELECT * FROM t0 WHERE 2.143 IS NULL;", TestEnvironment.getQueriesString(reducedResult)); + } + } From f852f8cc08a98db34d40d20011b7c8fe47abf453 Mon Sep 17 00:00:00 2001 From: Branimir Vujicic Date: Thu, 24 Aug 2023 17:52:20 +0200 Subject: [PATCH 389/856] Add support for Presto - common classes (#884) To implement support for Presto some changes in common classes are required. ## Changes in common classes: 1. class sqlancer.MainOptions - added global parameters : * canonicalizeString (boolean) - presto doesn't support JDBC queries with ";" at the end of statement * compareResultsContent (boolean) - comparing content of VARBINARY columns fails 2. class sqlancer.ComparatorHelper - compare result based on parameter ``` boolean compare = state.getOptions().compareResultsContent(); if (compare && !firstHashSet.equals(secondHashSet)) { ``` 3. sqlancer.common.query.SQLancerResultSet - added method: ``` public String getType(int i) throws SQLException { return rs.getMetaData().getColumnTypeName(i); } ``` 4. sqlancer.common.query.SQLQueryAdapter : added constructor ``` public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema, boolean canonicalizeString) { ``` --- src/sqlancer/ComparatorHelper.java | 6 +++-- src/sqlancer/MainOptions.java | 14 ++++++++++ .../common/query/SQLQueryAdapter.java | 11 +++++++- .../common/query/SQLancerResultSet.java | 4 +++ test/sqlancer/TestComparatorHelper.java | 26 ++++++++++++++++--- 5 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 41d2cf996..5da635de2 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -48,7 +48,8 @@ public static List getResultSetFirstColumnAsString(String queryString, E e.printStackTrace(); } } - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + boolean canonicalizeString = state.getOptions().canonicalizeSqlString(); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, true, canonicalizeString); List resultSet = new ArrayList<>(); SQLancerResultSet result = null; try { @@ -106,7 +107,8 @@ public static void assumeResultSetsAreEqual(List resultSet, List Set firstHashSet = new HashSet<>(resultSet); Set secondHashSet = new HashSet<>(secondResultSet); - if (!firstHashSet.equals(secondHashSet)) { + boolean validateResultSizeOnly = state.getOptions().validateResultSizeOnly(); + if (!validateResultSizeOnly && !firstHashSet.equals(secondHashSet)) { Set firstResultSetMisses = new HashSet<>(firstHashSet); firstResultSetMisses.removeAll(secondHashSet); Set secondResultSetMisses = new HashSet<>(secondHashSet); diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index 232856f94..a6a252fa3 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -141,6 +141,12 @@ public class MainOptions { @Parameter(names = "--ast-reducer-max-time", description = "EXPERIMENTAL Maximum time duration (secs) the statement reducer will do") private long maxStatementReduceTime = NO_REDUCE_LIMIT; // NOPMD + @Parameter(names = "--validate-result-size-only", description = "Should validate result size only and skip comparing content of the result set ", arity = 1) + private boolean validateResultSizeOnly = false; // NOPMD + + @Parameter(names = "--canonicalize-sql-strings", description = "Should canonicalize query string (add ';' at the end", arity = 1) + private boolean canonicalizeSqlString = true; // NOPMD + public int getMaxExpressionDepth() { return maxExpressionDepth; } @@ -322,4 +328,12 @@ public long getMaxASTReduceTime() { return maxASTReduceTime; } + public boolean validateResultSizeOnly() { + return validateResultSizeOnly; + } + + public boolean canonicalizeSqlString() { + return canonicalizeSqlString; + } + } diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 248403458..3784ed635 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -32,7 +32,16 @@ private static boolean guessAffectSchemaFromQuery(String query) { } public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema) { - this.query = canonicalizeString(query); + this(query, expectedErrors, couldAffectSchema, true); + } + + public SQLQueryAdapter(String query, ExpectedErrors expectedErrors, boolean couldAffectSchema, + boolean canonicalizeString) { + if (canonicalizeString) { + this.query = canonicalizeString(query); + } else { + this.query = query; + } this.expectedErrors = expectedErrors; this.couldAffectSchema = couldAffectSchema; checkQueryString(); diff --git a/src/sqlancer/common/query/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java index f83cdd3a5..d6f2a67cb 100644 --- a/src/sqlancer/common/query/SQLancerResultSet.java +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -46,6 +46,10 @@ public long getLong(int i) throws SQLException { return rs.getLong(i); } + public String getType(int i) throws SQLException { + return rs.getMetaData().getColumnTypeName(i); + } + public void registerEpilogue(Runnable runnableEpilogue) { this.runnableEpilogue = runnableEpilogue; } diff --git a/test/sqlancer/TestComparatorHelper.java b/test/sqlancer/TestComparatorHelper.java index 6815b9258..e8b06388b 100644 --- a/test/sqlancer/TestComparatorHelper.java +++ b/test/sqlancer/TestComparatorHelper.java @@ -2,19 +2,37 @@ import static org.junit.jupiter.api.Assertions.assertThrowsExactly; +import java.sql.SQLException; import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.Test; +import sqlancer.h2.H2Options; +import sqlancer.h2.H2Schema; + public class TestComparatorHelper { // TODO: Implement tests for the other ComparatorHelper methods + // TODO: create test state that not depends on specific database + final SQLGlobalState state = new SQLGlobalState() { + + @Override + protected H2Schema readSchema() throws SQLException { + return H2Schema.fromConnection(getConnection(), getDatabaseName()); + } + + @Override + public MainOptions getOptions() { + return new MainOptions(); + } + }; + @Test public void testAssumeResultSetsAreEqualWithEqualSets() { List r1 = Arrays.asList("a", "b", "c"); List r2 = Arrays.asList("a", "b", "c"); - ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state); } @@ -26,7 +44,7 @@ public void testAssumeResultSetsAreEqualWithUnequalLengthSets() { // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions // is raised assertThrowsExactly(NullPointerException.class, () -> { - ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state); }); } @@ -38,7 +56,7 @@ public void testAssumeResultSetsAreEqualWithUnequalValueSets() { // line occurs before AssertionError is thrown, but it's good enough as an indicator that one of the Exceptions // is raised assertThrowsExactly(NullPointerException.class, () -> { - ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null); + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state); }); } @@ -46,7 +64,7 @@ public void testAssumeResultSetsAreEqualWithUnequalValueSets() { public void testAssumeResultSetsAreEqualWithCanonicalizationRule() { List r1 = Arrays.asList("a", "b", "c"); List r2 = Arrays.asList("a", "b", "d"); - ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), null, (String s) -> { + ComparatorHelper.assumeResultSetsAreEqual(r1, r2, "", Arrays.asList(""), state, (String s) -> { return s.equals("d") ? "c" : s; }); } From 7804a3adec0962ad6d24687c42ec473aa49669fe Mon Sep 17 00:00:00 2001 From: Branimir Vujicic Date: Wed, 30 Aug 2023 03:41:55 +0200 Subject: [PATCH 390/856] Add support for Presto (#881) This is the first Pull Request for [Presto](https://prestodb.io/) implementation. The following oracles are currently completed: * NoRec oracle * TLP Where * TLP Aggregate (basic aggregate function) ## Data types The following data types are currently supported: BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, JSON, DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND, ARRAY. Others such as MAP, ROW, IPADDRESS, UID, IPPREFIX, HyperLogLog, P4HyperLogLog, KHyperLogLog, QDigest and TDigest will be supported in the future. ## Functions Large number of Presto supported are implemented, but only small number are actually used in test. Most of the functions will be supported in the future. --- README.md | 8 +- pom.xml | 7 +- src/check_names.py | 1 + src/sqlancer/Main.java | 2 + src/sqlancer/presto/PrestoConstantUtils.java | 40 + src/sqlancer/presto/PrestoErrors.java | 127 +++ .../presto/PrestoExpressionToNode.java | 25 + src/sqlancer/presto/PrestoGlobalState.java | 13 + src/sqlancer/presto/PrestoOptions.java | 166 ++++ src/sqlancer/presto/PrestoProvider.java | 195 +++++ src/sqlancer/presto/PrestoSchema.java | 484 +++++++++++ .../presto/PrestoToStringVisitor.java | 149 ++++ .../presto/ast/PrestoAggregateFunction.java | 714 +++++++++++++++ .../presto/ast/PrestoAtTimeZoneOperator.java | 22 + .../presto/ast/PrestoCastFunction.java | 24 + .../presto/ast/PrestoColumnReference.java | 12 + .../presto/ast/PrestoComparisonFunction.java | 81 ++ .../presto/ast/PrestoConditionalFunction.java | 95 ++ src/sqlancer/presto/ast/PrestoConstant.java | 810 ++++++++++++++++++ .../presto/ast/PrestoDateFunction.java | 523 +++++++++++ .../presto/ast/PrestoDefaultFunction.java | 233 +++++ src/sqlancer/presto/ast/PrestoExpression.java | 5 + src/sqlancer/presto/ast/PrestoFunction.java | 127 +++ .../ast/PrestoFunctionWithoutParenthesis.java | 24 + src/sqlancer/presto/ast/PrestoJoin.java | 118 +++ .../ast/PrestoMultiValuedComparison.java | 39 + .../PrestoMultiValuedComparisonOperator.java | 43 + .../ast/PrestoMultiValuedComparisonType.java | 11 + .../ast/PrestoQuantifiedComparison.java | 36 + src/sqlancer/presto/ast/PrestoSelect.java | 18 + .../presto/ast/PrestoTableReference.java | 11 + .../ast/PrestoUnaryPostfixOperation.java | 52 ++ .../ast/PrestoUnaryPrefixOperation.java | 62 ++ .../presto/gen/PrestoAlterTableGenerator.java | 67 ++ .../presto/gen/PrestoDeleteGenerator.java | 32 + .../presto/gen/PrestoIndexGenerator.java | 57 ++ .../presto/gen/PrestoInsertGenerator.java | 52 ++ .../gen/PrestoRandomQuerySynthesizer.java | 72 ++ .../presto/gen/PrestoTableGenerator.java | 69 ++ .../gen/PrestoTypedExpressionGenerator.java | 798 +++++++++++++++++ .../presto/gen/PrestoUpdateGenerator.java | 53 ++ .../presto/gen/PrestoViewGenerator.java | 36 + .../presto/test/PrestoNoRECOracle.java | 138 +++ ...restoQueryPartitioningAggregateTester.java | 206 +++++ .../test/PrestoQueryPartitioningBase.java | 90 ++ ...PrestoQueryPartitioningDistinctTester.java | 44 + .../PrestoQueryPartitioningGroupByTester.java | 53 ++ .../PrestoQueryPartitioningHavingTester.java | 64 ++ .../PrestoQueryPartitioningWhereTester.java | 46 + 49 files changed, 6150 insertions(+), 4 deletions(-) create mode 100644 src/sqlancer/presto/PrestoConstantUtils.java create mode 100644 src/sqlancer/presto/PrestoErrors.java create mode 100644 src/sqlancer/presto/PrestoExpressionToNode.java create mode 100644 src/sqlancer/presto/PrestoGlobalState.java create mode 100644 src/sqlancer/presto/PrestoOptions.java create mode 100644 src/sqlancer/presto/PrestoProvider.java create mode 100644 src/sqlancer/presto/PrestoSchema.java create mode 100644 src/sqlancer/presto/PrestoToStringVisitor.java create mode 100644 src/sqlancer/presto/ast/PrestoAggregateFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java create mode 100644 src/sqlancer/presto/ast/PrestoCastFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoColumnReference.java create mode 100644 src/sqlancer/presto/ast/PrestoComparisonFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoConditionalFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoConstant.java create mode 100644 src/sqlancer/presto/ast/PrestoDateFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoDefaultFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoExpression.java create mode 100644 src/sqlancer/presto/ast/PrestoFunction.java create mode 100644 src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java create mode 100644 src/sqlancer/presto/ast/PrestoJoin.java create mode 100644 src/sqlancer/presto/ast/PrestoMultiValuedComparison.java create mode 100644 src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java create mode 100644 src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java create mode 100644 src/sqlancer/presto/ast/PrestoQuantifiedComparison.java create mode 100644 src/sqlancer/presto/ast/PrestoSelect.java create mode 100644 src/sqlancer/presto/ast/PrestoTableReference.java create mode 100644 src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java create mode 100644 src/sqlancer/presto/gen/PrestoAlterTableGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoDeleteGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoIndexGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoInsertGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java create mode 100644 src/sqlancer/presto/gen/PrestoTableGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoUpdateGenerator.java create mode 100644 src/sqlancer/presto/gen/PrestoViewGenerator.java create mode 100644 src/sqlancer/presto/test/PrestoNoRECOracle.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningBase.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java create mode 100644 src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java diff --git a/README.md b/README.md index 5c18c887d..d84a993b1 100644 --- a/README.md +++ b/README.md @@ -68,11 +68,13 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | | OceanBase | Working | Untyped | | | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | -| Databend | Working | Typed | | -| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | +| Databend | Working | Typed | | +| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | | CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | | Materialize |Working | Typed | -| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. +| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. +| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. + # Using SQLancer diff --git a/pom.xml b/pom.xml index 4944369fa..292378933 100644 --- a/pom.xml +++ b/pom.xml @@ -316,6 +316,11 @@ duckdb_jdbc 0.5.1 + + com.facebook.presto + presto-jdbc + 0.283 + org.junit.jupiter junit-jupiter-engine @@ -467,4 +472,4 @@ - \ No newline at end of file + diff --git a/src/check_names.py b/src/check_names.py index a88c94e89..048da7669 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -48,6 +48,7 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["MySQL"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "mysql")) name_to_files["OceanBase"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "oceanbase")) name_to_files["Postgres"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "postgres")) + name_to_files["Presto"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "presto")) name_to_files["QuestDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "questdb")) name_to_files["SQLite3"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "sqlite3")) name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 4b480e595..89fbafd38 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -43,6 +43,7 @@ import sqlancer.mysql.MySQLProvider; import sqlancer.oceanbase.OceanBaseProvider; import sqlancer.postgres.PostgresProvider; +import sqlancer.presto.PrestoProvider; import sqlancer.questdb.QuestDBProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.stonedb.StoneDBProvider; @@ -719,6 +720,7 @@ private static void checkForIssue799(List> providers) providers.add(new MongoDBProvider()); providers.add(new MySQLProvider()); providers.add(new OceanBaseProvider()); + providers.add(new PrestoProvider()); providers.add(new PostgresProvider()); providers.add(new QuestDBProvider()); providers.add(new SQLite3Provider()); diff --git a/src/sqlancer/presto/PrestoConstantUtils.java b/src/sqlancer/presto/PrestoConstantUtils.java new file mode 100644 index 000000000..6de744019 --- /dev/null +++ b/src/sqlancer/presto/PrestoConstantUtils.java @@ -0,0 +1,40 @@ +package sqlancer.presto; + +import java.math.BigDecimal; +import java.math.RoundingMode; + +public final class PrestoConstantUtils { + + private PrestoConstantUtils() { + } + + public static String removeNoneAscii(String str) { + return str.replaceAll("[^\\x00-\\x7F]", ""); + } + + public static String removeNonePrintable(String str) { // All Control Char + return str.replaceAll("[\\p{C}]", ""); + } + + public static String removeOthersControlChar(String str) { // Some Control Char + return str.replaceAll("[\\p{Cntrl}\\p{Cc}\\p{Cf}\\p{Co}\\p{Cn}]", ""); + } + + public static String removeAllControlChars(String str) { + return removeOthersControlChar(removeNonePrintable(str)).replaceAll("[\\r\\n\\t]", ""); + } + + public static BigDecimal getDecimal(double val, int scale, int precision) { + int part = precision - scale; + // long part + long lng = (long) val; + // decimal places + double d1 = val - lng; + String xStr = Long.toString(lng); + String substring = xStr.substring(xStr.length() - part); + long newX = substring.isEmpty() ? 0 : Long.parseLong(substring); + double finalD = newX + d1; + return new BigDecimal(finalD).setScale(scale, RoundingMode.CEILING); + } + +} diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java new file mode 100644 index 000000000..603addb5c --- /dev/null +++ b/src/sqlancer/presto/PrestoErrors.java @@ -0,0 +1,127 @@ +package sqlancer.presto; + +import sqlancer.common.query.ExpectedErrors; + +public final class PrestoErrors { + + private PrestoErrors() { + } + + public static void addExpressionErrors(ExpectedErrors errors) { + // Presto errors + errors.add("cannot be applied to"); + errors.add("LIKE expression must evaluate to a varchar"); + errors.add("JOIN ON clause must evaluate to a boolean"); + // errors.add("Unexpected parameters"); + + // SELECT SUM(count) FROM (SELECT + // CAST((-179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0000 + // IS NOT NULL AND + // -179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0000) + // AS BIGINT)as count FROM t0) as res + errors.add("Decimal overflow"); + errors.add("multiplication overflow"); + errors.add("addition overflow"); + errors.add("subtraction overflow"); + + // cast + // errors.add("Cannot cast"); + errors.add("Value cannot be cast to"); + errors.add("Cannot cast DECIMAL"); + errors.add("Cannot cast BIGINT"); + errors.add("Cannot cast INTEGER"); + + // TODO: check + errors.add("io.airlift.slice.Slice cannot be cast to java.lang.Number"); + errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); + errors.add("Unexpected subquery expression in logical plan"); + + // 9223372036854775808 + errors.add("Invalid numeric literal"); + + errors.add("Division by zero"); + errors.add("/ by zero"); + + errors.add("Cannot subtract hour, minutes or seconds from a date"); + errors.add("Cannot add hour, minutes or seconds to a date"); + + errors.add("DECIMAL scale must be in range"); + errors.add("multiplication overflow"); + errors.add("addition overflow"); + errors.add("subtraction overflow"); + errors.add("Decimal overflow"); + errors.add("IN value and list items must be the same type"); + errors.add("is not a valid timestamp literal"); + errors.add("Unknown time-zone ID"); + errors.add("GROUP BY position"); + + // ARRAY + errors.add("Unknown type: ARRAY"); + } + + private static void addRegexErrors(ExpectedErrors errors) { + errors.add("missing ]"); + errors.add("missing )"); + errors.add("invalid escape sequence"); + errors.add("no argument for repetition operator: "); + errors.add("bad repetition operator"); + errors.add("trailing \\"); + errors.add("invalid perl operator"); + errors.add("invalid character class range"); + errors.add("width is not integer"); + } + + private static void addFunctionErrors(ExpectedErrors errors) { + errors.add("SUBSTRING cannot handle negative lengths"); + errors.add("is undefined outside [-1,1]"); // ACOS etc + errors.add("invalid type specifier"); // PRINTF + errors.add("argument index out of range"); // PRINTF + errors.add("invalid format string"); // PRINTF + errors.add("number is too big"); // PRINTF + errors.add("Like pattern must not end with escape character!"); // LIKE + errors.add("Could not choose a best candidate function for the function call \"date_part"); // date_part + errors.add("extract specifier"); // date_part + errors.add("not recognized"); // date_part + errors.add("not supported"); // date_part + errors.add("Failed to cast"); + errors.add("Conversion Error"); + errors.add("Could not cast value"); + errors.add("Insufficient padding in RPAD"); // RPAD + errors.add("Could not choose a best candidate function for the function call"); // monthname + errors.add("expected a numeric precision field"); // ROUND + errors.add("with non-constant precision is not supported"); // ROUND + } + + // TODO: cover presto error + public static void addInsertErrors(ExpectedErrors errors) { + addRegexErrors(errors); + addFunctionErrors(errors); + + errors.add("NOT NULL constraint failed"); + errors.add("PRIMARY KEY or UNIQUE constraint violated"); + errors.add("duplicate key"); + errors.add("can't be cast because the value is out of range for the destination type"); + errors.add("Could not convert string"); + errors.add("Unimplemented type for cast"); + errors.add("field value out of range"); + errors.add("CHECK constraint failed"); + errors.add("Cannot explicitly insert values into rowid column"); // TODO: don't insert into rowid + errors.add(" Column with name rowid does not exist!"); // currently, there doesn't seem to way to determine if + // the table has a primary key + errors.add("Could not cast value"); + errors.add("create unique index, table contains duplicate data"); + errors.add("Failed to cast"); + + errors.add("Values rows have mismatched types"); + errors.add("Mismatch at column"); + errors.add("This connector does not support updates or deletes"); + errors.add("Values rows have mismatched types"); + errors.add("Invalid numeric literal"); + + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.add("must be an aggregate expression or appear in GROUP BY clause"); + } + +} diff --git a/src/sqlancer/presto/PrestoExpressionToNode.java b/src/sqlancer/presto/PrestoExpressionToNode.java new file mode 100644 index 000000000..96497e47c --- /dev/null +++ b/src/sqlancer/presto/PrestoExpressionToNode.java @@ -0,0 +1,25 @@ +package sqlancer.presto; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.ast.PrestoExpression; + +public final class PrestoExpressionToNode { + + private PrestoExpressionToNode() { + + } + + @SuppressWarnings("unchecked") + public static Node cast(PrestoExpression expression) { + return (Node) expression; + } + + @SuppressWarnings("unchecked") + public static List> casts(List expressions) { + return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/presto/PrestoGlobalState.java b/src/sqlancer/presto/PrestoGlobalState.java new file mode 100644 index 000000000..eb053712e --- /dev/null +++ b/src/sqlancer/presto/PrestoGlobalState.java @@ -0,0 +1,13 @@ +package sqlancer.presto; + +import java.sql.SQLException; + +import sqlancer.SQLGlobalState; + +public class PrestoGlobalState extends SQLGlobalState { + + @Override + protected PrestoSchema readSchema() throws SQLException { + return PrestoSchema.fromConnection(getConnection(), getDatabaseName()); + } +} diff --git a/src/sqlancer/presto/PrestoOptions.java b/src/sqlancer/presto/PrestoOptions.java new file mode 100644 index 000000000..78deab27c --- /dev/null +++ b/src/sqlancer/presto/PrestoOptions.java @@ -0,0 +1,166 @@ +package sqlancer.presto; + +import java.util.ArrayList; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.presto.test.PrestoNoRECOracle; +import sqlancer.presto.test.PrestoQueryPartitioningAggregateTester; +import sqlancer.presto.test.PrestoQueryPartitioningDistinctTester; +import sqlancer.presto.test.PrestoQueryPartitioningGroupByTester; +import sqlancer.presto.test.PrestoQueryPartitioningHavingTester; +import sqlancer.presto.test.PrestoQueryPartitioningWhereTester; + +@Parameters(commandDescription = "Presto") +public class PrestoOptions implements DBMSSpecificOptions { + + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 8080; + + @Parameter(names = "--test-collate", arity = 1) + public boolean testCollate = true; + + @Parameter(names = "--test-check", description = "Allow generating CHECK constraints in tables", arity = 1) + public boolean testCheckConstraints = true; + + @Parameter(names = "--test-default-values", description = "Allow generating DEFAULT values in tables", arity = 1) + public boolean testDefaultValues = true; + + @Parameter(names = "--test-not-null", description = "Allow generating NOT NULL constraints in tables", arity = 1) + public boolean testNotNullConstraints = true; + + @Parameter(names = "--test-functions", description = "Allow generating functions in expressions", arity = 1) + public boolean testFunctions = true; + + @Parameter(names = "--test-casts", description = "Allow generating casts in expressions", arity = 1) + public boolean testCasts = true; + + @Parameter(names = "--test-between", description = "Allow generating the BETWEEN operator in expressions (FALSE by default : Presto null handling in BETWEEN operator : https://prestodb.io/docs/current/functions/comparison.html )", arity = 1) + public boolean testBetween; + + @Parameter(names = "--test-in", description = "Allow generating the IN operator in expressions", arity = 1) + public boolean testIn = true; + + @Parameter(names = "--test-case", description = "Allow generating the CASE operator in expressions", arity = 1) + public boolean testCase = true; + + @Parameter(names = "--test-binary-logicals", description = "Allow generating AND and OR in expressions", arity = 1) + public boolean testBinaryLogicals = true; + + @Parameter(names = "--test-int-constants", description = "Allow generating INTEGER constants", arity = 1) + public boolean testIntConstants = true; + + @Parameter(names = "--test-varchar-constants", description = "Allow generating VARCHAR constants", arity = 1) + public boolean testStringConstants = true; + + @Parameter(names = "--test-time-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testDateConstants = true; + + @Parameter(names = "--test-date-constants", description = "Allow generating DATE constants", arity = 1) + public boolean testTimeConstants = true; + + @Parameter(names = "--test-timestamp-constants", description = "Allow generating TIMESTAMP constants", arity = 1) + public boolean testTimestampConstants = true; + + @Parameter(names = "--test-float-constants", description = "Allow generating floating-point constants", arity = 1) + public boolean testFloatConstants = true; + + @Parameter(names = "--test-boolean-constants", description = "Allow generating boolean constants", arity = 1) + public boolean testBooleanConstants = true; + + @Parameter(names = "--test-binary-comparisons", description = "Allow generating binary comparison operators (e.g., >= or LIKE)", arity = 1) + public boolean testBinaryComparisons = true; + + @Parameter(names = "--test-indexes", description = "Allow explicit (i.e. CREATE INDEX) and implicit (i.e., UNIQUE and PRIMARY KEY) indexes", arity = 1) + public boolean testIndexes = true; + + @Parameter(names = "--test-rowid", description = "Test tables' rowid columns", arity = 1) + public boolean testRowid = true; + + @Parameter(names = "--max-num-views", description = "The maximum number of views that can be generated for a database", arity = 1) + public int maxNumViews = 1; + + @Parameter(names = "--max-num-deletes", description = "The maximum number of DELETE statements that are issued for a database", arity = 1) + public int maxNumDeletes = 1; + + @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) + public int maxNumUpdates = 5; + + @Parameter(names = "--oracle") + public List oracles = List.of(PrestoOracleFactory.NOREC); + + @Parameter(names = "--catalog") + public String catalog = "memory"; + + @Parameter(names = "--schema") + public String schema = "test"; + + @Parameter(names = "--typed-generator", description = "the expression generator type - typed and untyped ") + public boolean typedGenerator = true; + + @Override + public List getTestOracleFactory() { + return oracles; + } + + public enum PrestoOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoNoRECOracle(globalState); + } + + }, + HAVING { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PrestoGlobalState globalState) { + List> oracles = new ArrayList<>(); + oracles.add(new PrestoQueryPartitioningWhereTester(globalState)); + oracles.add(new PrestoQueryPartitioningHavingTester(globalState)); + oracles.add(new PrestoQueryPartitioningAggregateTester(globalState)); + oracles.add(new PrestoQueryPartitioningDistinctTester(globalState)); + oracles.add(new PrestoQueryPartitioningGroupByTester(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + } + + } + +} diff --git a/src/sqlancer/presto/PrestoProvider.java b/src/sqlancer/presto/PrestoProvider.java new file mode 100644 index 000000000..be3ef4325 --- /dev/null +++ b/src/sqlancer/presto/PrestoProvider.java @@ -0,0 +1,195 @@ +package sqlancer.presto; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.presto.gen.PrestoInsertGenerator; +import sqlancer.presto.gen.PrestoTableGenerator; + +@AutoService(DatabaseProvider.class) +public class PrestoProvider extends SQLProviderAdapter { + + public PrestoProvider() { + super(PrestoGlobalState.class, PrestoOptions.class); + } + + // TODO : check actions based on connector + // returns number of actions + private static int mapActions(PrestoGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + if (Objects.requireNonNull(a) == Action.INSERT) { + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + // case UPDATE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumUpdates + 1); + // case EXPLAIN: + // return r.getInteger(0, 2); + // case DELETE: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumDeletes + 1); + // case CREATE_VIEW: + // return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumViews + 1); + } + throw new AssertionError(a); + } + + @Override + public void generateDatabase(PrestoGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + SQLQueryAdapter qt = new PrestoTableGenerator().getQuery(globalState); + success = globalState.executeStatement(qt); + } while (!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + PrestoProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(PrestoGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + boolean useSSl = true; + if (globalState.getOptions().isDefaultUsername() && globalState.getOptions().isDefaultPassword()) { + username = "presto"; + password = null; + useSSl = false; + } + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = PrestoOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = PrestoOptions.DEFAULT_PORT; + } + String catalogName = globalState.getDbmsSpecificOptions().catalog; + String databaseName = globalState.getDatabaseName(); + String url = String.format("jdbc:presto://%s:%d/%s?SSL=%b", host, port, catalogName, useSSl); + Connection con = DriverManager.getConnection(url, username, password); + List schemaNames = getSchemaNames(con, catalogName, databaseName); + dropExistingTables(con, catalogName, databaseName, schemaNames); + dropSchema(globalState, con, catalogName, databaseName); + createSchema(globalState, con, catalogName, databaseName); + useSchema(globalState, con, catalogName, databaseName); + return new SQLConnection(con); + + } + + private static void useSchema(PrestoGlobalState globalState, Connection con, String catalogName, + String databaseName) throws SQLException { + globalState.getState().logStatement("USE " + catalogName + "." + databaseName); + try (Statement s = con.createStatement()) { + s.execute("USE " + catalogName + "." + databaseName); + } + } + + private static void createSchema(PrestoGlobalState globalState, Connection con, String catalogName, + String databaseName) throws SQLException { + globalState.getState().logStatement("CREATE SCHEMA IF NOT EXISTS " + catalogName + "." + databaseName); + try (Statement s = con.createStatement()) { + s.execute("CREATE SCHEMA IF NOT EXISTS " + catalogName + "." + databaseName); + } + } + + private static void dropSchema(PrestoGlobalState globalState, Connection con, String catalogName, + String databaseName) throws SQLException { + globalState.getState().logStatement("DROP SCHEMA IF EXISTS " + catalogName + "." + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP SCHEMA IF EXISTS " + catalogName + "." + databaseName); + } + } + + private static List getSchemaNames(Connection con, String catalogName, String databaseName) + throws SQLException { + List schemaNames = new ArrayList<>(); + final String showSchemasSql = "SHOW SCHEMAS FROM " + catalogName + " LIKE '" + databaseName + "'"; + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(showSchemasSql)) { + while (rs.next()) { + schemaNames.add(rs.getString("Schema")); + } + } + } + return schemaNames; + } + + private static void dropExistingTables(Connection con, String catalogName, String databaseName, + List schemaNames) throws SQLException { + if (!schemaNames.isEmpty()) { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("SHOW TABLES FROM " + catalogName + "." + databaseName)) { + while (rs.next()) { + tableNames.add(rs.getString("Table")); + } + } + } + try (Statement s = con.createStatement()) { + for (String tableName : tableNames) { + s.execute("DROP TABLE IF EXISTS " + catalogName + "." + databaseName + "." + tableName); + } + } + } + } + + @Override + public String getDBMSName() { + return "presto"; + } + + public enum Action implements AbstractAction { + // SHOW_TABLES((g) -> new SQLQueryAdapter("SHOW TABLES", new ExpectedErrors(), false, false)), // + INSERT(PrestoInsertGenerator::getQuery); + // TODO : check actions based on connector + // DELETE(PrestoDeleteGenerator::generate), // + // UPDATE(PrestoUpdateGenerator::getQuery), // + // CREATE_VIEW(PrestoViewGenerator::generate), // + // EXPLAIN((g) -> { + // ExpectedErrors errors = new ExpectedErrors(); + // PrestoErrors.addExpressionErrors(errors); + // PrestoErrors.addGroupByErrors(errors); + // return new SQLQueryAdapter( + // "EXPLAIN " + PrestoToStringVisitor + // .asString(PrestoRandomQuerySynthesizer.generateSelect(g, Randomly.smallNumber() + 1)), + // errors); + // }); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(PrestoGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + +} diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java new file mode 100644 index 000000000..70323a18b --- /dev/null +++ b/src/sqlancer/presto/PrestoSchema.java @@ -0,0 +1,484 @@ +package sqlancer.presto; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; + +public class PrestoSchema extends AbstractSchema { + + public PrestoSchema(List databaseTables) { + super(databaseTables); + } + + public static PrestoSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, databaseName, tableName); + boolean isView = tableName.startsWith("v"); + PrestoTable t = new PrestoTable(tableName, databaseColumns, isView); + for (PrestoColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new PrestoSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + // TODO: UPDATE + // SHOW TABLES [ FROM schema ] [ LIKE pattern [ ESCAPE 'escape_character' ] ] + try (ResultSet rs = s.executeQuery("SHOW TABLES")) { + while (rs.next()) { + tableNames.add(rs.getString("Table")); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String databaseName, String tableName) + throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("select " + " table_catalog " + " , table_schema " + + " , table_name " + " , column_name " + " , is_nullable " + " , data_type " + + " from information_schema.columns " + " where table_schema = '%s' and table_name = '%s'", + databaseName, tableName))) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + boolean isNullable = rs.getString("is_nullable").contentEquals("YES"); + PrestoColumn c = new PrestoColumn(columnName, getColumnType(dataType), false, isNullable); + columns.add(c); + } + } + } + + return columns; + } + + private static PrestoCompositeDataType getColumnType(String typeString) { + int bracesStart = typeString.indexOf('('); + String type; + int size = 0; + int precision = 0; + if (bracesStart != -1) { + type = typeString.substring(0, bracesStart); + } else { + type = typeString; + } + type = type.toUpperCase(); + + PrestoDataType primitiveType; + switch (type) { + case "INTEGER": + primitiveType = PrestoDataType.INT; + size = 4; + break; + case "SMALLINT": + primitiveType = PrestoDataType.INT; + size = 2; + break; + case "BIGINT": + primitiveType = PrestoDataType.INT; + size = 8; + break; + case "TINYINT": + primitiveType = PrestoDataType.INT; + size = 1; + break; + case "VARCHAR": + primitiveType = PrestoDataType.VARCHAR; + break; + case "VARBINARY": + primitiveType = PrestoDataType.VARBINARY; + break; + case "CHAR": + primitiveType = PrestoDataType.CHAR; + break; + case "FLOAT": + case "REAL": + primitiveType = PrestoDataType.FLOAT; + size = 4; + break; + case "DOUBLE": + primitiveType = PrestoDataType.FLOAT; + size = 8; + break; + case "DECIMAL": + primitiveType = PrestoDataType.DECIMAL; + break; + case "BOOLEAN": + primitiveType = PrestoDataType.BOOLEAN; + break; + case "DATE": + primitiveType = PrestoDataType.DATE; + break; + case "TIME": + primitiveType = PrestoDataType.TIME; + break; + case "TIME WITH TIME ZONE": + primitiveType = PrestoDataType.TIME_WITH_TIME_ZONE; + break; + case "TIMESTAMP": + primitiveType = PrestoDataType.TIMESTAMP; + break; + case "TIMESTAMP WITH TIME ZONE": + primitiveType = PrestoDataType.TIMESTAMP_WITH_TIME_ZONE; + break; + case "INTERVAL DAY TO SECOND": + primitiveType = PrestoDataType.INTERVAL_DAY_TO_SECOND; + break; + case "INTERVAL YEAR TO MONTH": + primitiveType = PrestoDataType.INTERVAL_YEAR_TO_MONTH; + break; + case "JSON": + primitiveType = PrestoDataType.JSON; + break; + case "ARRAY": + primitiveType = PrestoDataType.ARRAY; + break; + case "NULL": + primitiveType = PrestoDataType.NULL; + break; + default: + throw new AssertionError(typeString); + } + return new PrestoCompositeDataType(primitiveType, size, precision); + } + + public PrestoTables getRandomTableNonEmptyTables() { + return new PrestoTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + + public enum PrestoDataType { + BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, JSON, DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, + TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND, ARRAY, + // MAP, + // ROW, + // IPADDRESS, + // UID, + // IPPREFIX, + // HyperLogLog, + // P4HyperLogLog, + // KHyperLogLog, + // QDigest, + // TDigest, + NULL; + + public static PrestoDataType getRandomWithoutNull() { + PrestoDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == PrestoDataType.NULL); + return dt; + } + + public static List getNumericTypes() { + return Arrays.asList(INT, FLOAT, DECIMAL, DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, + TIMESTAMP_WITH_TIME_ZONE); + } + + public static List getComparableTypes() { + return Arrays.asList(BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, JSON, DATE, TIME, TIMESTAMP, + TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND); + } + + public static List getOrderableTypes() { + return Arrays.asList(BOOLEAN, INT, FLOAT, DECIMAL, VARCHAR, CHAR, VARBINARY, + // JSON, + DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE, INTERVAL_YEAR_TO_MONTH, + INTERVAL_DAY_TO_SECOND, ARRAY); + } + + public static List getNumberTypes() { + return Arrays.asList(INT, FLOAT, DECIMAL); + } + + public static List getTemporalTypes() { + return Arrays.asList(DATE, TIME, TIMESTAMP, TIME_WITH_TIME_ZONE, TIMESTAMP_WITH_TIME_ZONE); + } + + public static List getIntervalTypes() { + return Arrays.asList(INTERVAL_YEAR_TO_MONTH, INTERVAL_DAY_TO_SECOND); + } + + public static List getTextTypes() { + return Arrays.asList(VARCHAR, CHAR, VARBINARY, JSON); + } + + public boolean isNumeric() { + switch (this) { + case INT: + case FLOAT: + case DECIMAL: + return true; + default: + return false; + } + } + + public boolean isOrderable() { + return getOrderableTypes().contains(this); + } + + public PrestoCompositeDataType get() { + return PrestoCompositeDataType.fromDataType(this); + } + } + + public static class PrestoCompositeDataType { + + private final PrestoDataType dataType; + + private final int size; + + private final int scale; + + private final PrestoCompositeDataType elementType; + + public PrestoCompositeDataType(PrestoDataType dataType, int dataSize, int dataScale) { + this.dataType = dataType; + this.size = dataSize; + this.scale = dataScale; + this.elementType = null; + } + + public PrestoCompositeDataType(PrestoDataType dataType, PrestoCompositeDataType elementType) { + if (dataType != PrestoDataType.ARRAY) { + throw new IllegalArgumentException(); + } + this.dataType = dataType; + this.size = -1; + this.scale = -1; + this.elementType = elementType; + } + + public static PrestoCompositeDataType getRandomWithoutNull() { + PrestoDataType type = PrestoDataType.getRandomWithoutNull(); + int size; + int scale = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case DECIMAL: + size = Math.toIntExact(8); + scale = Math.toIntExact(4); + break; + case VARBINARY: + case JSON: + case VARCHAR: + case CHAR: + size = Math.toIntExact(Randomly.getNotCachedInteger(10, 250)); + break; + case ARRAY: + return new PrestoCompositeDataType(type, PrestoCompositeDataType.getRandomWithoutNull()); + case BOOLEAN: + case DATE: + case TIME: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP: + case TIMESTAMP_WITH_TIME_ZONE: + case INTERVAL_DAY_TO_SECOND: + case INTERVAL_YEAR_TO_MONTH: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new PrestoCompositeDataType(type, size, scale); + } + + public static PrestoCompositeDataType fromDataType(PrestoDataType type) { + int size; + int scale = -1; + switch (type) { + case INT: + size = Randomly.fromOptions(1, 2, 4, 8); + break; + case FLOAT: + size = Randomly.fromOptions(4, 8); + break; + case DECIMAL: + size = Math.toIntExact(8); + scale = Math.toIntExact(4); + break; + case JSON: + case VARCHAR: + case CHAR: + size = Math.toIntExact(Randomly.getNotCachedInteger(10, 250)); + break; + case ARRAY: + return new PrestoCompositeDataType(type, PrestoCompositeDataType.getRandomWithoutNull()); + case BOOLEAN: + case VARBINARY: + case DATE: + case TIME: + case TIMESTAMP: + case TIMESTAMP_WITH_TIME_ZONE: + case TIME_WITH_TIME_ZONE: + case INTERVAL_DAY_TO_SECOND: + case INTERVAL_YEAR_TO_MONTH: + size = 0; + break; + default: + throw new AssertionError(type); + } + + return new PrestoCompositeDataType(type, size, scale); + } + + public PrestoDataType getPrimitiveDataType() { + return dataType; + } + + public int getSize() { + if (size == -1) { + throw new AssertionError(this); + } + return size; + } + + public int getScale() { + if (scale == -1) { + throw new AssertionError(this); + } + return scale; + } + + @Override + public String toString() { + switch (getPrimitiveDataType()) { + case INT: + switch (size) { + case 8: + return "BIGINT"; + case 4: + return "INTEGER"; + case 2: + return "SMALLINT"; + case 1: + return "TINYINT"; + default: + throw new AssertionError(size); + } + case VARBINARY: + return "VARBINARY"; + case JSON: + return "JSON"; + case VARCHAR: + return "VARCHAR" + "(" + size + ")"; + case CHAR: + return "CHAR" + "(" + size + ")"; + case FLOAT: + switch (size) { + case 4: + return "REAL"; + case 8: + return "DOUBLE"; + default: + throw new AssertionError(size); + } + case DECIMAL: + return "DECIMAL" + "(" + size + ", " + scale + ")"; + case BOOLEAN: + return "BOOLEAN"; + case TIMESTAMP_WITH_TIME_ZONE: + return "TIMESTAMP WITH TIME ZONE"; + case TIMESTAMP: + return "TIMESTAMP"; + case INTERVAL_YEAR_TO_MONTH: + return "INTERVAL YEAR TO MONTH"; + case INTERVAL_DAY_TO_SECOND: + return "INTERVAL DAY TO SECOND"; + case DATE: + return "DATE"; + case TIME: + return "TIME"; + case TIME_WITH_TIME_ZONE: + return "TIME WITH TIME ZONE"; + case ARRAY: + return "ARRAY(" + elementType + ")"; + case NULL: + return "NULL"; + default: + throw new AssertionError(getPrimitiveDataType()); + } + } + + public PrestoCompositeDataType getElementType() { + return elementType; + } + + public boolean isOrderable() { + if (dataType == PrestoDataType.ARRAY) { + assert elementType != null; + return elementType.isOrderable(); + } + return dataType.isOrderable(); + } + + } + + public static class PrestoColumn extends AbstractTableColumn { + + private final boolean isPrimaryKey; + private final boolean isNullable; + + public PrestoColumn(String name, PrestoCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable) { + super(name, null, columnType); + this.isPrimaryKey = isPrimaryKey; + this.isNullable = isNullable; + } + + public boolean isPrimaryKey() { + return isPrimaryKey; + } + + public boolean isNullable() { + return isNullable; + } + + public boolean isOrderable() { + return getType().getPrimitiveDataType().isOrderable(); + } + + } + + public static class PrestoTables extends AbstractTables { + + public PrestoTables(List tables) { + super(tables); + } + + } + + public static class PrestoTable extends AbstractRelationalTable { + + public PrestoTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + } + +} diff --git a/src/sqlancer/presto/PrestoToStringVisitor.java b/src/sqlancer/presto/PrestoToStringVisitor.java new file mode 100644 index 000000000..1342ff635 --- /dev/null +++ b/src/sqlancer/presto/PrestoToStringVisitor.java @@ -0,0 +1,149 @@ +package sqlancer.presto; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.ast.PrestoAtTimeZoneOperator; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoConstant; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoFunctionWithoutParenthesis; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoMultiValuedComparison; +import sqlancer.presto.ast.PrestoQuantifiedComparison; +import sqlancer.presto.ast.PrestoSelect; + +public class PrestoToStringVisitor extends NewToStringVisitor { + + public static String asString(Node expr) { + PrestoToStringVisitor visitor = new PrestoToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof PrestoConstant) { + visit((PrestoConstant) expr); + } else if (expr instanceof PrestoSelect) { + visit((PrestoSelect) expr); + } else if (expr instanceof PrestoJoin) { + visit((PrestoJoin) expr); + } else if (expr instanceof PrestoCastFunction) { + visit((PrestoCastFunction) expr); + } else if (expr instanceof PrestoFunctionWithoutParenthesis) { + visit((PrestoFunctionWithoutParenthesis) expr); + } else if (expr instanceof PrestoAtTimeZoneOperator) { + visit((PrestoAtTimeZoneOperator) expr); + } else if (expr instanceof PrestoMultiValuedComparison) { + visit((PrestoMultiValuedComparison) expr); + } else if (expr instanceof PrestoQuantifiedComparison) { + visit((PrestoQuantifiedComparison) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(PrestoJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + if (join.getOuterType() != null) { + sb.append(join.getOuterType()); + } + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(PrestoConstant constant) { + sb.append(constant.toString()); + } + + private void visit(PrestoAtTimeZoneOperator timeZoneOperator) { + visit(timeZoneOperator.getExpr()); + sb.append(" AT TIME ZONE "); + sb.append(timeZoneOperator.getTimeZone()); + } + + private void visit(PrestoFunctionWithoutParenthesis prestoFunctionWithoutParenthesis) { + sb.append(prestoFunctionWithoutParenthesis.getExpr()); + } + + private void visit(PrestoSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByExpressions().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByExpressions()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + public void visit(PrestoCastFunction cast) { + sb.append("CAST(("); + visit(cast.getExpr()); + sb.append(") AS "); + sb.append(cast.getType().toString()); + sb.append(")"); + } + + public void visit(PrestoMultiValuedComparison comp) { + sb.append("("); + visit(comp.getLeft()); + sb.append(" "); + sb.append(comp.getOp().getStringRepresentation()); + sb.append(" "); + sb.append(comp.getType()); + sb.append(" (VALUES "); + visit(comp.getRight()); + sb.append(")"); + sb.append(")"); + } + + public void visit(PrestoQuantifiedComparison comp) { + sb.append("("); + visit(comp.getLeft()); + sb.append(" "); + sb.append(comp.getOp().getStringRepresentation()); + sb.append(" "); + sb.append(comp.getType()); + sb.append(" ( "); + visit(comp.getRight()); + sb.append(" ) "); + sb.append(")"); + } +} diff --git a/src/sqlancer/presto/ast/PrestoAggregateFunction.java b/src/sqlancer/presto/ast/PrestoAggregateFunction.java new file mode 100644 index 000000000..0b118393f --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoAggregateFunction.java @@ -0,0 +1,714 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public enum PrestoAggregateFunction implements PrestoFunction { + + // General Aggregate Functions + + // arbitrary(x) → [same as input] + // Returns an arbitrary non-null value of x, if one exists. + ARBITRARY("arbitrary", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return PrestoDataType.getRandomWithoutNull(); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + + // TODO: + // + // array_agg(x) → array<[same as input]># + // Returns an array created from the input x elements. + + // avg(x) → double + // Returns the average (arithmetic mean) of all input values. + AVG("avg", PrestoDataType.FLOAT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { + Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL) }; + } + }, + // avg(time interval type) → time interval type# + // Returns the average interval length of all input values. + AVG_INTERVAL_YM("avg", PrestoDataType.INTERVAL_YEAR_TO_MONTH, PrestoDataType.INTERVAL_YEAR_TO_MONTH), + AVG_INTERVAL_DS("avg", PrestoDataType.INTERVAL_DAY_TO_SECOND, PrestoDataType.INTERVAL_DAY_TO_SECOND), + + // bool_and(boolean) → boolean# + // Returns TRUE if every input value is TRUE, otherwise FALSE. + BOOL_AND("bool_and", PrestoDataType.BOOLEAN, PrestoDataType.BOOLEAN), + // bool_or(boolean) → boolean# + // Returns TRUE if any input value is TRUE, otherwise FALSE. + BOOL_OR("bool_or", PrestoDataType.BOOLEAN, PrestoDataType.BOOLEAN), + // checksum(x) → varbinary# + // Returns an order-insensitive checksum of the given values. + CHECKSUM("checksum", PrestoDataType.VARBINARY) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromList(PrestoDataType.getComparableTypes()) }; + } + }, + + // count(*) → bigint# + // Returns the number of input rows. + COUNT_ALL("count(*)", PrestoDataType.INT), + // count(x) → bigint# + // Returns the number of non-null input values. + COUNT_NOARGS("count", PrestoDataType.INT), COUNT("count", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.getRandomWithoutNull()) }; + } + }, + // count_if(x) → bigint# + // Returns the number of TRUE input values. This function is equivalent to count(CASE WHEN x THEN 1 END). + COUNT_IF("count_if", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.getRandomWithoutNull()) }; + } + }, + // every(boolean) → boolean# + // This is an alias for bool_and(). + EVERY("every", PrestoDataType.BOOLEAN, PrestoDataType.BOOLEAN), + // geometric_mean(x) → double# + // Returns the geometric mean of all input values. + GEOMETRIC_MEAN("geometric_mean", PrestoDataType.FLOAT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { + Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL) }; + } + }, + // max_by(x, y) → [same as x]# + // Returns the value of x associated with the maximum value of y over all input values. + MAX_BY("max_by", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), + Randomly.fromList(PrestoDataType.getOrderableTypes()) }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + + // TODO: + // + // max_by(x, y, n) → array<[same as x]># + // Returns n values of x associated with the n largest of all input values of y in descending order of y. + + // min_by(x, y) → [same as x]# + // Returns the value of x associated with the minimum value of y over all input values. + MIN_BY("min_by", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), + Randomly.fromList(PrestoDataType.getOrderableTypes()) }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + // TODO: + // + // min_by(x, y, n) → array<[same as x]> + // Returns n values of x associated with the n smallest of all input values of y in ascending order of y. + + // max(x) → [same as input] + // Returns the maximum value of all input values. + MAX("max", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + boolean isCompatible = PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + if (returnType.getPrimitiveDataType() == PrestoDataType.ARRAY && returnType.toString().contains("JSON")) { + isCompatible = false; + } + return isCompatible; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public PrestoCompositeDataType getCompositeReturnType() { + PrestoDataType dataType = Randomly.fromList(PrestoDataType.getOrderableTypes()); + PrestoCompositeDataType returnType; + do { + returnType = PrestoCompositeDataType.fromDataType(dataType); + } while (!isCompatibleWithReturnType(returnType)); + return returnType; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, true); + } + + }, + + // TODO: + // + // max(x, n) → array<[same as x]># + // Returns n largest values of all input values of x. + + // min(x) → [same as input]# + // Returns the minimum value of all input values. + MIN("min", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + boolean orderable = PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + if (returnType.getPrimitiveDataType() == PrestoDataType.ARRAY && returnType.toString().contains("JSON")) { + orderable = false; + } + return orderable; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromList(PrestoDataType.getOrderableTypes()); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + + }, + + // TODO: + // + // min(x, n) → array<[same as x]># + // Returns n smallest values of all input values of x. + + // TODO: + // + // reduce_agg(inputValue T, initialState S, inputFunction(S, T, S), combineFunction(S, S, S)) → S# + // Reduces all input values into a single value. inputFunction will be invoked for each input value. In addition to + // taking the input value, inputFunction takes the current state, initially initialState, and returns the new state. + // combineFunction will be invoked to combine two states into a new state. The final state is returned: + // + // SELECT id, reduce_agg(value, (a, b) -> a + b, (a, b) -> a + b) + // FROM ( + // VALUES + // (1, 2), + // (1, 3), + // (1, 4), + // (2, 20), + // (2, 30), + // (2, 40) + // ) AS t(id, value) + // GROUP BY id; + // -- (1, 9) + // -- (2, 90) + // + // SELECT id, reduce_agg(value, (a, b) -> a * b, (a, b) -> a * b) + // FROM ( + // VALUES + // (1, 2), + // (1, 3), + // (1, 4), + // (2, 20), + // (2, 30), + // (2, 40) + // ) AS t(id, value) + // GROUP BY id; + // -- (1, 24) + // -- (2, 24000) + // The state type must be a boolean, integer, floating-point, or date/time/interval. + + // TODO: + // + // set_agg(x) → array<[same as input]># + // Returns an array created from the distinct input x elements. + + // TODO: + // + // set_union(array(T)) -> array(T)# + // Returns an array of all the distinct values contained in each array of the input + // + // Example: + // + // SELECT set_union(elements) + // FROM ( + // VALUES + // ARRAY[1, 3], + // ARRAY[2, 4] + // ) AS t(elements); + // Returns ARRAY[1, 3, 4] + + // sum(x) → [same as input]# + // Returns the sum of all input values. + SUM("sum", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL) + .contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.DECIMAL); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + // sum(time interval type) → time interval type# + // Returns the average interval length of all input values. + SUM_INTERVAL_YM("sum", PrestoDataType.INTERVAL_YEAR_TO_MONTH, PrestoDataType.INTERVAL_YEAR_TO_MONTH), + SUM_INTERVAL_DS("sum", PrestoDataType.INTERVAL_DAY_TO_SECOND, PrestoDataType.INTERVAL_DAY_TO_SECOND), + + // Bitwise Aggregate Functions# + + // bitwise_and_agg(x) → bigint# + // Returns the bitwise AND of all input values in 2’s complement representation. + BITWISE_AND_AGG("bitwise_and_agg", PrestoDataType.INT, PrestoDataType.INT), + + // bitwise_or_agg(x) → bigint# + // Returns the bitwise OR of all input values in 2’s complement representation. + BITWISE_OR_AGG("bitwise_or_agg", PrestoDataType.INT, PrestoDataType.INT), + + // TODO: + // + // Map Aggregate Functions + + // histogram(x)# + // Returns a map containing the count of the number of times each input value occurs. + // + // map_agg(key, value)# + // Returns a map created from the input key / value pairs. + // + // map_union(x(K, V)) -> map(K, V)# + // Returns the union of all the input maps. If a key is found in multiple input maps, that key’s value in the + // resulting map comes from an arbitrary input map. + // + // map_union_sum(x(K, V)) -> map(K, V)# + // Returns the union of all the input maps summing the values of matching keys in all the maps. All null values in + // the original maps are coalesced to 0. + // + // multimap_agg(key, value)# + // Returns a multimap created from the input key / value pairs. Each key can be associated with multiple values. + + // Approximate Aggregate Functions# + // approx_distinct(x) → bigint# + // Returns the approximate number of distinct input values. This function provides an approximation of + // count(DISTINCT x). + // Zero is returned if all input values are null. + // This function should produce a standard error of 2.3%, which is the standard deviation of the (approximately + // normal) + // error distribution over all possible sets. It does not guarantee an upper bound on the error for any specific + // input set. + APPROX_DISTINCT("approx_distinct", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromList(PrestoDataType.getOrderableTypes()) }; + } + }, + // + // approx_distinct(x, e) → bigint# + // Returns the approximate number of distinct input values. This function provides an approximation of + // count(DISTINCT x). Zero is returned if all input values are null. + // + // This function should produce a standard error of no more than e, which is the standard deviation of the + // (approximately normal) error distribution over all possible sets. It does not guarantee an upper bound on the + // error for any specific input set. The current implementation of this function requires that e be in the range of + // [0.0040625, 0.26000]. + APPROX_DISTINCT_2("approx_distinct", PrestoDataType.INT) { + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromList(PrestoDataType.getOrderableTypes()), PrestoDataType.FLOAT }; + } + }, + // approx_percentile(x, percentage) → [same as x]# + // Returns the approximate percentile for all input values of x at the given percentage. + // The value of percentage must be between zero and one and must be constant for all input rows. + APPROX_PERCENTILE("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.FLOAT }; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List> arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType.fromDataType(getReturnType())); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + + // approx_percentile(x, percentage, accuracy) → [same as x]# + // As approx_percentile(x, percentage), but with a maximum rank error of accuracy. + // The value of accuracy must be between zero and one (exclusive) and must be constant for all input rows. + // Note that a lower “accuracy” is really a lower error threshold, and thus more accurate. The default accuracy is + // 0.01. + APPROX_PERCENTILE_ACCURACY("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.FLOAT, PrestoDataType.FLOAT }; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List> arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoFloatConstant(0.01D)); + } else { + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + } + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType + .fromDataType(Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT))); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + + // TODO: + // + // approx_percentile(x, percentages) → array<[same as x]># + // Returns the approximate percentile for all input values of x at each of the specified percentages. Each element + // of the percentages array must be between zero and one, and the array must be constant for all input rows. + // + // approx_percentile(x, percentages, accuracy) → array<[same as x]># + // As approx_percentile(x, percentages), but with a maximum rank error of accuracy. + + // approx_percentile(x, w, percentage) → [same as x]# + // Returns the approximate weighed percentile for all input values of x using the per-item weight w at the + // percentage p. + // The weight must be an integer value of at least one. + // It is effectively a replication count for the value x in the percentile set. + // The value of p must be between zero and one and must be constant for all input rows. + APPROX_PERCENTILE_WEIGHT("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.INT, PrestoDataType.FLOAT }; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List> arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoIntConstant(1)); + } else { + arguments.add(new PrestoConstant.PrestoIntConstant(Randomly.smallNumber())); + } + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType + .fromDataType(Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT))); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }, + + // approx_percentile(x, w, percentage, accuracy) → [same as x]# + // As approx_percentile(x, w, percentage), but with a maximum rank error of accuracy. + APPROX_PERCENTILE_PERCENTAGE_ACCURACY("approx_percentile", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return List.of(PrestoDataType.INT, PrestoDataType.FLOAT).contains(returnType.getPrimitiveDataType()); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT), + PrestoDataType.INT, PrestoDataType.FLOAT, PrestoDataType.FLOAT }; + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { + List> arguments = new ArrayList<>(); + arguments.add(gen.generateExpression(returnType2, depth + 1)); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoIntConstant(1)); + } else { + arguments.add(new PrestoConstant.PrestoIntConstant(Randomly.smallNumber())); + } + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + if (Randomly.getBooleanWithRatherLowProbability()) { + arguments.add(new PrestoConstant.PrestoFloatConstant(0.01D)); + } else { + arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); + } + return arguments; + } + + @Override + public PrestoDataType getReturnType() { + return Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoCompositeDataType returnType, boolean orderable) { + PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, + () -> PrestoCompositeDataType + .fromDataType(Randomly.fromOptions(PrestoDataType.INT, PrestoDataType.FLOAT))); + return super.getArgumentsForReturnType(gen, depth, returnTypeLocal, orderable); + } + }; + + // TODO: + // + // approx_percentile(x, w, percentages) → array<[same as x]># + // Returns the approximate weighed percentile for all input values of x using the per-item weight w at each of the + // given percentages specified in the array. The weight must be an integer value of at least one. It is effectively + // a replication count for the value x in the percentile set. Each element of the array must be between zero and + // one, and the array must be constant for all input rows. + // + // approx_percentile(x, w, percentages, accuracy) → array<[same as x]># + // As approx_percentile(x, w, percentages), but with a maximum rank error of accuracy. + // + // approx_set(x) → HyperLogLog + // See HyperLogLog Functions. + // + // merge(x) → HyperLogLog + // See HyperLogLog Functions. + // + // khyperloglog_agg(x) → KHyperLogLog + // See KHyperLogLog Functions. + + // TODO: + // + // merge(qdigest(T)) -> qdigest(T) + // See Quantile Digest Functions. + // + // qdigest_agg(x) → qdigest<[same as x]> + // See Quantile Digest Functions. + // + // qdigest_agg(x, w) → qdigest<[same as x]> + // See Quantile Digest Functions. + // + // qdigest_agg(x, w, accuracy) → qdigest<[same as x]> + // See Quantile Digest Functions. + // + // numeric_histogram(buckets, value, weight) → map# + // Computes an approximate histogram with up to buckets number of buckets for all values with a per-item weight of + // weight. + // The keys of the returned map are roughly the center of the bin, and the entry is the total weight of the bin. + // The algorithm is based loosely on [BenHaimTomTov2010]. + // + // buckets must be a bigint. value and weight must be numeric. + // + // numeric_histogram(buckets, value) → map# + // Computes an approximate histogram with up to buckets number of buckets for all values. This function is + // equivalent to the variant of numeric_histogram() that takes a weight, with a per-item weight of 1. In this case, + // the total weight in the returned map is the count of items in the bin. + + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoAggregateFunction(String functionName, PrestoDataType returnType) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = new PrestoDataType[0]; + } + + PrestoAggregateFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + public static PrestoAggregateFunction getRandomMetamorphicOracle() { + return Randomly.fromOptions(ARBITRARY, AVG, AVG_INTERVAL_YM, AVG_INTERVAL_DS, BOOL_AND, BOOL_OR, CHECKSUM, + COUNT_ALL, COUNT_NOARGS, COUNT, COUNT_IF, EVERY, GEOMETRIC_MEAN, MAX_BY, MIN_BY, MAX, MIN, SUM, + SUM_INTERVAL_YM, SUM_INTERVAL_DS, BITWISE_AND_AGG, BITWISE_OR_AGG); + } + + public static PrestoAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public static List getFunctionsCompatibleWith(PrestoCompositeDataType returnType) { + return Stream.of(values()).filter(f -> f.isCompatibleWithReturnType(returnType)).collect(Collectors.toList()); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + + @Override + public int getNumberOfArguments() { + return 1; + } + + public List getReturnTypes(PrestoSchema.PrestoDataType dataType) { + return Collections.singletonList(dataType); + } + + public PrestoDataType getReturnType() { + if (returnType == null) { + return PrestoDataType.getRandomWithoutNull(); + } + return returnType; + } + + public PrestoCompositeDataType getCompositeReturnType() { + PrestoDataType dataType = getReturnType(); + return PrestoCompositeDataType.fromDataType(dataType); + } +} diff --git a/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java new file mode 100644 index 000000000..7bb01bb98 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java @@ -0,0 +1,22 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.Node; + +public class PrestoAtTimeZoneOperator implements Node { + + private final Node expr; + private final Node timeZone; + + public PrestoAtTimeZoneOperator(Node expr, Node timeZone) { + this.expr = expr; + this.timeZone = timeZone; + } + + public Node getExpr() { + return expr; + } + + public Node getTimeZone() { + return timeZone; + } +} diff --git a/src/sqlancer/presto/ast/PrestoCastFunction.java b/src/sqlancer/presto/ast/PrestoCastFunction.java new file mode 100644 index 000000000..89b139314 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoCastFunction.java @@ -0,0 +1,24 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; + +public class PrestoCastFunction implements Node { + + private final Node expr; + private final PrestoSchema.PrestoCompositeDataType type; + + public PrestoCastFunction(Node expr, PrestoSchema.PrestoCompositeDataType type) { + this.expr = expr; + this.type = type; + } + + public Node getExpr() { + return expr; + } + + public PrestoSchema.PrestoCompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoColumnReference.java b/src/sqlancer/presto/ast/PrestoColumnReference.java new file mode 100644 index 000000000..4f96a013d --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.presto.PrestoSchema; + +public class PrestoColumnReference extends ColumnReferenceNode { + + public PrestoColumnReference(PrestoSchema.PrestoColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoComparisonFunction.java b/src/sqlancer/presto/ast/PrestoComparisonFunction.java new file mode 100644 index 000000000..1125c3e35 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoComparisonFunction.java @@ -0,0 +1,81 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; + +public enum PrestoComparisonFunction implements PrestoFunction { + + // comparison + + // Returns the largest of the provided values. + // → [same as input] + GREATEST("greatest", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return -1; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + }, + // Returns the smallest of the provided values. + // → [same as input]# + LEAST("least", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return -1; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + ArrayList prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + }; + + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoComparisonFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoConditionalFunction.java b/src/sqlancer/presto/ast/PrestoConditionalFunction.java new file mode 100644 index 000000000..e603a7e5d --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoConditionalFunction.java @@ -0,0 +1,95 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; + +public enum PrestoConditionalFunction implements PrestoFunction { + + // Conditional functions + IF_TRUE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { PrestoDataType.BOOLEAN, returnType.getPrimitiveDataType() }; + } + }, + + IF_TRUE_FALSE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { PrestoDataType.BOOLEAN, returnType.getPrimitiveDataType(), + returnType.getPrimitiveDataType() }; + } + }, + + NULLIF("nullif", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), returnType.getPrimitiveDataType() }; + } + }, + + COALESCE("coalesce", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return true; + } + + @Override + public int getNumberOfArguments() { + return -1; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + List prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + }; + + private final PrestoDataType returnType; + private final String functionName; + + PrestoConditionalFunction(String functionName, PrestoDataType returnType) { + this.functionName = functionName; + this.returnType = returnType; + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public int getNumberOfArguments() { + return getArgumentTypes(PrestoCompositeDataType.fromDataType(returnType)).length; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoConstant.java b/src/sqlancer/presto/ast/PrestoConstant.java new file mode 100644 index 000000000..97b115a4e --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoConstant.java @@ -0,0 +1,810 @@ +package sqlancer.presto.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.DecimalFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoConstantUtils; +import sqlancer.presto.PrestoSchema; + +public abstract class PrestoConstant implements Node, PrestoExpression { + + private static final String[] TIME_ZONES = { "Africa/Abidjan", "Africa/Accra", "Africa/Addis_Ababa", + "Africa/Algiers", "Africa/Asmara", "Africa/Asmera", "Africa/Bamako", "Africa/Bangui", "Africa/Banjul", + "Africa/Bissau", "Africa/Blantyre", "Africa/Brazzaville", "Africa/Bujumbura", "Africa/Cairo", + "Africa/Casablanca", "Africa/Ceuta", "Africa/Conakry", "Africa/Dakar", "Africa/Dar_es_Salaam", + "Africa/Djibouti", "Africa/Douala", "Africa/El_Aaiun", "Africa/Freetown", "Africa/Gaborone", + "Africa/Harare", "Africa/Johannesburg", "Africa/Juba", "Africa/Kampala", "Africa/Khartoum", "Africa/Kigali", + "Africa/Kinshasa", "Africa/Lagos", "Africa/Libreville", "Africa/Lome", "Africa/Luanda", "Africa/Lubumbashi", + "Africa/Lusaka", "Africa/Malabo", "Africa/Maputo", "Africa/Maseru", "Africa/Mbabane", "Africa/Mogadishu", + "Africa/Monrovia", "Africa/Nairobi", "Africa/Ndjamena", "Africa/Niamey", "Africa/Nouakchott", + "Africa/Ouagadougou", "Africa/Porto-Novo", "Africa/Sao_Tome", "Africa/Timbuktu", "Africa/Tripoli", + "Africa/Tunis", "Africa/Windhoek", "America/Adak", "America/Anchorage", "America/Anguilla", + "America/Antigua", "America/Araguaina", "America/Argentina/Buenos_Aires", "America/Argentina/Catamarca", + "America/Argentina/ComodRivadavia", "America/Argentina/Cordoba", "America/Argentina/Jujuy", + "America/Argentina/La_Rioja", "America/Argentina/Mendoza", "America/Argentina/Rio_Gallegos", + "America/Argentina/Salta", "America/Argentina/San_Juan", "America/Argentina/San_Luis", + "America/Argentina/Tucuman", "America/Argentina/Ushuaia", "America/Aruba", "America/Asuncion", + "America/Atikokan", "America/Atka", "America/Bahia", "America/Barbados", "America/Belem", "America/Belize", + "America/Blanc-Sablon", "America/Boa_Vista", "America/Bogota", "America/Boise", "America/Buenos_Aires", + "America/Cambridge_Bay", "America/Campo_Grande", "America/Cancun", "America/Caracas", "America/Catamarca", + "America/Cayenne", "America/Cayman", "America/Chicago", "America/Chihuahua", "America/Coral_Harbour", + "America/Cordoba", "America/Costa_Rica", "America/Creston", "America/Cuiaba", "America/Curacao", + "America/Danmarkshavn", "America/Dawson", "America/Dawson_Creek", "America/Denver", "America/Detroit", + "America/Dominica", "America/Edmonton", "America/Eirunepe", "America/El_Salvador", "America/Ensenada", + "America/Fort_Nelson", "America/Fort_Wayne", "America/Fortaleza", "America/Glace_Bay", "America/Godthab", + "America/Goose_Bay", "America/Grand_Turk", "America/Grenada", "America/Guadeloupe", "America/Guatemala", + "America/Guayaquil", "America/Guyana", "America/Halifax", "America/Havana", "America/Hermosillo", + "America/Indiana/Indianapolis", "America/Indiana/Knox", "America/Indiana/Marengo", + "America/Indiana/Petersburg", "America/Indiana/Tell_City", "America/Indiana/Vevay", + "America/Indiana/Vincennes", "America/Indiana/Winamac", "America/Indianapolis", "America/Inuvik", + "America/Iqaluit", "America/Jamaica", "America/Jujuy", "America/Juneau", "America/Kentucky/Louisville", + "America/Kentucky/Monticello", "America/Knox_IN", "America/Kralendijk", "America/La_Paz", "America/Lima", + "America/Los_Angeles", "America/Louisville", "America/Lower_Princes", "America/Maceio", "America/Managua", + "America/Manaus", "America/Marigot", "America/Martinique", "America/Matamoros", "America/Mendoza", + "America/Menominee", "America/Merida", "America/Metlakatla", "America/Mexico_City", "America/Miquelon", + "America/Moncton", "America/Monterrey", "America/Montevideo", "America/Montreal", "America/Montserrat", + "America/Nassau", "America/New_York", "America/Nipigon", "America/Nome", "America/Noronha", + "America/North_Dakota/Beulah", "America/North_Dakota/Center", "America/North_Dakota/New_Salem", + "America/Nuuk", "America/Ojinaga", "America/Panama", "America/Pangnirtung", "America/Paramaribo", + "America/Phoenix", "America/Port-au-Prince", "America/Port_of_Spain", "America/Porto_Acre", + "America/Porto_Velho", "America/Puerto_Rico", "America/Punta_Arenas", "America/Rainy_River", + "America/Rankin_Inlet", "America/Recife", "America/Regina", "America/Resolute", "America/Rio_Branco", + "America/Rosario", "America/Santa_Isabel", "America/Santarem", "America/Santiago", "America/Santo_Domingo", + "America/Sao_Paulo", "America/Scoresbysund", "America/Shiprock", "America/Sitka", "America/St_Barthelemy", + "America/St_Johns", "America/St_Kitts", "America/St_Lucia", "America/St_Thomas", "America/St_Vincent", + "America/Swift_Current", "America/Tegucigalpa", "America/Thule", "America/Thunder_Bay", "America/Tijuana", + "America/Toronto", "America/Tortola", "America/Vancouver", "America/Virgin", "America/Whitehorse", + "America/Winnipeg", "America/Yakutat", "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis", + "Antarctica/DumontDUrville", "Antarctica/Macquarie", "Antarctica/Mawson", "Antarctica/McMurdo", + "Antarctica/Palmer", "Antarctica/Rothera", "Antarctica/South_Pole", "Antarctica/Syowa", "Antarctica/Troll", + "Antarctica/Vostok", "Arctic/Longyearbyen", "Asia/Aden", "Asia/Almaty", "Asia/Amman", "Asia/Anadyr", + "Asia/Aqtau", "Asia/Aqtobe", "Asia/Ashgabat", "Asia/Ashkhabad", "Asia/Atyrau", "Asia/Baghdad", + "Asia/Bahrain", "Asia/Baku", "Asia/Bangkok", "Asia/Barnaul", "Asia/Beirut", "Asia/Bishkek", "Asia/Brunei", + "Asia/Calcutta", "Asia/Chita", "Asia/Choibalsan", "Asia/Chongqing", "Asia/Chungking", "Asia/Colombo", + "Asia/Dacca", "Asia/Dhaka", "Asia/Dili", "Asia/Dubai", "Asia/Dushanbe", "Asia/Famagusta", "Asia/Gaza", + "Asia/Harbin", "Asia/Hebron", "Asia/Ho_Chi_Minh", "Asia/Hong_Kong", "Asia/Hovd", "Asia/Irkutsk", + "Asia/Istanbul", "Asia/Jakarta", "Asia/Jayapura", "Asia/Jerusalem", "Asia/Kabul", "Asia/Kamchatka", + "Asia/Karachi", "Asia/Kashgar", "Asia/Kathmandu", "Asia/Katmandu", "Asia/Khandyga", "Asia/Kolkata", + "Asia/Krasnoyarsk", "Asia/Kuala_Lumpur", "Asia/Kuching", "Asia/Kuwait", "Asia/Macao", "Asia/Macau", + "Asia/Magadan", "Asia/Makassar", "Asia/Manila", "Asia/Muscat", "Asia/Nicosia", "Asia/Novokuznetsk", + "Asia/Novosibirsk", "Asia/Omsk", "Asia/Oral", "Asia/Phnom_Penh", "Asia/Pontianak", "Asia/Pyongyang", + "Asia/Qatar", "Asia/Qostanay", "Asia/Qyzylorda", "Asia/Rangoon", "Asia/Riyadh", "Asia/Saigon", + "Asia/Sakhalin", "Asia/Samarkand", "Asia/Seoul", "Asia/Shanghai", "Asia/Singapore", "Asia/Srednekolymsk", + "Asia/Taipei", "Asia/Tashkent", "Asia/Tbilisi", "Asia/Tehran", "Asia/Tel_Aviv", "Asia/Thimbu", + "Asia/Thimphu", "Asia/Tokyo", "Asia/Tomsk", "Asia/Ujung_Pandang", "Asia/Ulaanbaatar", "Asia/Ulan_Bator", + "Asia/Urumqi", "Asia/Ust-Nera", "Asia/Vientiane", "Asia/Vladivostok", "Asia/Yakutsk", "Asia/Yangon", + "Asia/Yekaterinburg", "Asia/Yerevan", "Atlantic/Azores", "Atlantic/Bermuda", "Atlantic/Canary", + "Atlantic/Cape_Verde", "Atlantic/Faeroe", "Atlantic/Faroe", "Atlantic/Jan_Mayen", "Atlantic/Madeira", + "Atlantic/Reykjavik", "Atlantic/South_Georgia", "Atlantic/St_Helena", "Atlantic/Stanley", "Australia/ACT", + "Australia/Adelaide", "Australia/Brisbane", "Australia/Broken_Hill", "Australia/Canberra", + "Australia/Currie", "Australia/Darwin", "Australia/Eucla", "Australia/Hobart", "Australia/LHI", + "Australia/Lindeman", "Australia/Lord_Howe", "Australia/Melbourne", "Australia/North", "Australia/Perth", + "Australia/Queensland", "Australia/South", "Australia/Sydney", "Australia/Tasmania", "Australia/Victoria", + "Australia/West", "Australia/Yancowinna", "Brazil/Acre", "Brazil/DeNoronha", "Brazil/East", "Brazil/West", + "CET", "CST6CDT", "Canada/Atlantic", "Canada/Central", "Canada/Eastern", "Canada/Mountain", + "Canada/Newfoundland", "Canada/Pacific", "Canada/Saskatchewan", "Canada/Yukon", "Chile/Continental", + "Chile/EasterIsland", "Cuba", "EET", "EST5EDT", "Egypt", "Eire", "Etc/GMT", "Etc/GMT+0", "Etc/GMT+1", + "Etc/GMT+10", "Etc/GMT+11", "Etc/GMT+12", "Etc/GMT+2", "Etc/GMT+3", "Etc/GMT+4", "Etc/GMT+5", "Etc/GMT+6", + "Etc/GMT+7", "Etc/GMT+8", "Etc/GMT+9", "Etc/GMT-0", "Etc/GMT-1", "Etc/GMT-10", "Etc/GMT-11", "Etc/GMT-12", + "Etc/GMT-13", "Etc/GMT-14", "Etc/GMT-2", "Etc/GMT-3", "Etc/GMT-4", "Etc/GMT-5", "Etc/GMT-6", "Etc/GMT-7", + "Etc/GMT-8", "Etc/GMT-9", "Etc/GMT0", "Etc/Greenwich", "Etc/UCT", "Etc/UTC", "Etc/Universal", "Etc/Zulu", + "Europe/Amsterdam", "Europe/Andorra", "Europe/Astrakhan", "Europe/Athens", "Europe/Belfast", + "Europe/Belgrade", "Europe/Berlin", "Europe/Bratislava", "Europe/Brussels", "Europe/Bucharest", + "Europe/Budapest", "Europe/Busingen", "Europe/Chisinau", "Europe/Copenhagen", "Europe/Dublin", + "Europe/Gibraltar", "Europe/Guernsey", "Europe/Helsinki", "Europe/Isle_of_Man", "Europe/Istanbul", + "Europe/Jersey", "Europe/Kaliningrad", "Europe/Kiev", "Europe/Kirov", "Europe/Lisbon", "Europe/Ljubljana", + "Europe/London", "Europe/Luxembourg", "Europe/Madrid", "Europe/Malta", "Europe/Mariehamn", "Europe/Minsk", + "Europe/Monaco", "Europe/Moscow", "Europe/Nicosia", "Europe/Oslo", "Europe/Paris", "Europe/Podgorica", + "Europe/Prague", "Europe/Riga", "Europe/Rome", "Europe/Samara", "Europe/San_Marino", "Europe/Sarajevo", + "Europe/Saratov", "Europe/Simferopol", "Europe/Skopje", "Europe/Sofia", "Europe/Stockholm", "Europe/Tirane", + "Europe/Tiraspol", "Europe/Ulyanovsk", "Europe/Uzhgorod", "Europe/Vaduz", "Europe/Vatican", "Europe/Vienna", + "Europe/Vilnius", "Europe/Volgograd", "Europe/Warsaw", "Europe/Zagreb", "Europe/Zaporozhye", + "Europe/Zurich", "GB", "GB-Eire", "GMT", "GMT0", "Greenwich", "Hongkong", "Iceland", "Indian/Antananarivo", + "Indian/Chagos", "Indian/Christmas", "Indian/Cocos", "Indian/Comoro", "Indian/Kerguelen", "Indian/Mahe", + "Indian/Maldives", "Indian/Mauritius", "Indian/Mayotte", "Indian/Reunion", "Iran", "Israel", "Jamaica", + "Japan", "Kwajalein", "Libya", "MET", "MST7MDT", "Mexico/General", "NZ", "NZ-CHAT", "Navajo", "PRC", + "PST8PDT", "Pacific/Apia", "Pacific/Auckland", "Pacific/Bougainville", "Pacific/Chatham", "Pacific/Chuuk", + "Pacific/Easter", "Pacific/Efate", "Pacific/Enderbury", "Pacific/Fakaofo", "Pacific/Fiji", + "Pacific/Funafuti", "Pacific/Galapagos", "Pacific/Gambier", "Pacific/Guadalcanal", "Pacific/Guam", + "Pacific/Honolulu", "Pacific/Johnston", "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein", + "Pacific/Majuro", "Pacific/Marquesas", "Pacific/Midway", "Pacific/Nauru", "Pacific/Niue", "Pacific/Norfolk", + "Pacific/Noumea", "Pacific/Pago_Pago", "Pacific/Palau", "Pacific/Pitcairn", "Pacific/Pohnpei", + "Pacific/Ponape", "Pacific/Port_Moresby", "Pacific/Rarotonga", "Pacific/Saipan", "Pacific/Samoa", + "Pacific/Tahiti", "Pacific/Tarawa", "Pacific/Tongatapu", "Pacific/Truk", "Pacific/Wake", "Pacific/Wallis", + "Pacific/Yap", "Poland", "Portugal", "ROK", "Singapore", "Turkey", "UCT", "US/Alaska", "US/Aleutian", + "US/Arizona", "US/Central", "US/East-Indiana", "US/Eastern", "US/Hawaii", "US/Indiana-Starke", + "US/Michigan", "US/Mountain", "US/Pacific", "US/Samoa", "UTC", "Universal", "W-SU", "WET", "Zulu" }; + private static final String FALSE = "false"; + private static final String TRUE = "true"; + + private PrestoConstant() { + } + + public static Node createStringConstant(String text) { + return new PrestoTextConstant(text); + } + + public static Node createStringConstant(String text, int size) { + return new PrestoTextConstant(text, size); + } + + public static Node createJsonConstant() { + return new PrestoJsonConstant(); + } + + public static Node createFloatConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + assert type.getSize() == 4; + float floatValue = (float) val; + return new PrestoFloatConstant(floatValue); + } + + public static Node createFloatConstant(double val) { + return new PrestoFloatConstant(val); + } + + public static Node createDecimalConstant(double val) { + return new PrestoDecimalConstant(val); + } + + public static Node createDecimalConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + int scale = type.getScale(); + int precision = type.getSize(); + BigDecimal finalBD = PrestoConstantUtils.getDecimal(val, scale, precision); + return new PrestoDecimalConstant(finalBD.doubleValue()); + } + + public static Node createIntConstant(long val) { + return new PrestoIntConstant(val); + } + + public static Node createIntConstant(PrestoSchema.PrestoCompositeDataType type, long val, + boolean castInteger) { + PrestoIntConstant intConstant; + assert List.of(1, 2, 4, 8).contains(type.getSize()); + switch (type.getSize()) { + case 1: + intConstant = new PrestoIntConstant((byte) val); + break; + case 2: + intConstant = new PrestoIntConstant((short) val); + break; + case 4: + intConstant = new PrestoIntConstant((int) val); + break; + default: + intConstant = new PrestoIntConstant(val); + } + if (castInteger) { + return new PrestoCastFunction(intConstant, type); + } else { + return intConstant; + } + } + + public static Node createNullConstant() { + return new PrestoNullConstant(); + } + + public static Node createBooleanConstant(boolean val) { + return new PrestoBooleanConstant(val); + } + + public static Node createDateConstant(long integer) { + return new PrestoDateConstant(integer); + } + + public static Node createTimeConstant(long integer) { + return new PrestoTimeConstant(integer); + } + + public static Node createTimeWithTimeZoneConstant(long integer) { + return new PrestoTimeWithTimeZoneConstant(integer); + } + + public static Node createTimestampWithTimeZoneConstant(long integer) { + return new PrestoTimestampWithTimezoneConstant(integer); + } + + public static Node createIntervalDayToSecond(long integer) { + return new PrestoIntervalDayToSecondConstant(); + } + + public static Node createIntervalYearToMonth(long integer) { + return new PrestoIntervalYearToMonthConstant(); + } + + public static Node createTimestampConstant(long integer) { + return new PrestoTimestampConstant(integer); + } + + public static Node createVarbinaryConstant(String string) { + return new PrestoVarbinaryConstant(string); + } + + public static Node createTimezoneConstant() { + String string = Randomly.fromOptions(TIME_ZONES); + return new PrestoTextConstant(string); + } + + public static Node createArrayConstant(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); + long size = Randomly.getNotCachedInteger(0, 10); + + List> elements = new ArrayList<>(); + for (int i = 0; i <= size; i++) { + if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + elements.add(createArrayConstant(elementType)); + } else { + elements.add(generateConstant(elementType, false)); + } + } + return new PrestoArrayConstant(elements); + } + + public static Node createMapConstant(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); + long size = Randomly.getNotCachedInteger(0, 10); + + List> elements = new ArrayList<>(); + for (int i = 0; i <= size; i++) { + if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + elements.add(createArrayConstant(elementType)); + } else { + elements.add(generateConstant(elementType, false)); + } + } + return new PrestoArrayConstant(elements); + } + + public static Node generateConstant(PrestoSchema.PrestoCompositeDataType type, + boolean castInteger) { + Randomly randomly = new Randomly(); + switch (type.getPrimitiveDataType()) { + case ARRAY: + return PrestoConstant.createArrayConstant(type); + case NULL: + return PrestoConstant.createNullConstant(); + case CHAR: + return PrestoConstant.PrestoTextConstant.createStringConstant(randomly.getAlphabeticChar(), type.getSize()); + case VARCHAR: + return PrestoConstant.PrestoTextConstant.createStringConstant(randomly.getString(), type.getSize()); + case VARBINARY: + return PrestoConstant.createVarbinaryConstant(randomly.getString()); + case JSON: + return PrestoConstant.PrestoJsonConstant.createJsonConstant(); + case TIME: + return PrestoConstant.createTimeConstant(randomly.getLong(0, System.currentTimeMillis())); + case TIME_WITH_TIME_ZONE: + return PrestoConstant.createTimeWithTimeZoneConstant(randomly.getLong(0, System.currentTimeMillis())); + case TIMESTAMP: + return PrestoConstant.createTimestampConstant(randomly.getLong(0, System.currentTimeMillis())); + case TIMESTAMP_WITH_TIME_ZONE: + return PrestoConstant.createTimestampWithTimeZoneConstant(randomly.getLong(0, System.currentTimeMillis())); + case INTERVAL_YEAR_TO_MONTH: + return PrestoConstant.createIntervalYearToMonth(randomly.getLong(0, System.currentTimeMillis())); + case INTERVAL_DAY_TO_SECOND: + return PrestoConstant.createIntervalDayToSecond(randomly.getLong(0, System.currentTimeMillis())); + case INT: + return PrestoConstant.PrestoIntConstant.createIntConstant(type, Randomly.getNonCachedInteger(), + castInteger); + case FLOAT: + return PrestoConstant.PrestoFloatConstant.createFloatConstant(randomly.getDouble()); + case BOOLEAN: + return PrestoConstant.PrestoBooleanConstant.createBooleanConstant(Randomly.getBoolean()); + case DATE: + return PrestoConstant.createDateConstant(randomly.getLong(0, System.currentTimeMillis())); + case DECIMAL: + return PrestoConstant.createDecimalConstant(type, randomly.getLong(0, System.currentTimeMillis())); + default: + throw new AssertionError("Unknown type: " + type); + } + } + + public boolean isNull() { + return false; + } + + public boolean isInt() { + return false; + } + + public boolean isBoolean() { + return false; + } + + public boolean isArray() { + return false; + } + + public boolean isString() { + return false; + } + + public boolean isFloat() { + return false; + } + + public boolean asBoolean() { + throw new UnsupportedOperationException(this.toString()); + } + + public long asInt() { + throw new UnsupportedOperationException(this.toString()); + } + + public String asString() { + throw new UnsupportedOperationException(this.toString()); + } + + public double asFloat() { + throw new UnsupportedOperationException(this.toString()); + } + + public static class PrestoNullConstant extends PrestoConstant { + + @Override + public String toString() { + return "NULL"; + } + + @Override + public boolean isNull() { + return true; + } + + } + + public static class PrestoIntConstant extends PrestoConstant { + + private final long value; + + public PrestoIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + @Override + public boolean isInt() { + return true; + } + + } + + public static class PrestoFloatConstant extends PrestoConstant { + + private final double value; + + public PrestoFloatConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "infinity()"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "-infinity()"; + } + return String.valueOf(value); + } + + @Override + public boolean isFloat() { + return true; + } + + @Override + public double asFloat() { + return value; + } + + } + + public static class PrestoDecimalConstant extends PrestoConstant { + + private static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("###0.0000"); + + private final double value; + + public PrestoDecimalConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return DECIMAL_FORMAT.format(value); + } + + @Override + public double asFloat() { + return value; + } + + } + + public static class PrestoTextConstant extends PrestoConstant { + + private final String value; + + public PrestoTextConstant(String value) { + this.value = value; + } + + public PrestoTextConstant(String value, int size) { + this.value = value.substring(0, Math.min(value.length(), size)); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + + } + + public static class PrestoVarbinaryConstant extends PrestoConstant { + + private final String value; + + public PrestoVarbinaryConstant(String value) { + this.value = value.replace("'", ""); + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return String.format("CAST ('%s' AS VARBINARY)", value); + } + + } + + public static class PrestoJsonConstant extends PrestoConstant { + + private final String value; + + public PrestoJsonConstant() { + Randomly rand = new Randomly(); + JsonValueType jvt = Randomly.fromOptions(JsonValueType.values()); + String val; + switch (jvt) { + case NULL: + val = "null"; + value = "{\"val\":" + val + "}"; + break; + case FALSE: + val = FALSE; + value = "{\"val\":" + val + "}"; + break; + case TRUE: + val = TRUE; + value = "{\"val\":" + val + "}"; + break; + case STRING: + String randString = rand.getString(); + String string = randString.substring(0, Math.min(randString.length(), 250)); + string = string.replace("'", ""); + // https://www.rfc-editor.org/rfc/rfc8259#page-8 + string = PrestoConstantUtils.removeAllControlChars(string); + string = string.replace("\\", "\\\\"); + + value = "{\"val\": \"" + string + "\"}"; + break; + case NUMBER: + if (Randomly.getBoolean()) { + int no = (int) rand.getInteger(); + val = String.valueOf(no); + } else { + double no = rand.getDouble(); + val = String.valueOf(no); + } + value = "{\"val\": " + val + "}"; + break; + case ARRAY: + value = "{\"employees\":[\"John\", \"Anna\", \"Peter\"]}"; + break; + case OBJECT: + value = "{\"employee\":{\"name\":\"John\", \"age\":30, \"city\":\"New York\"}}"; + break; + default: + value = "{}"; + } + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "JSON '" + value + "'"; + } + + private enum JsonValueType { + OBJECT, ARRAY, NUMBER, STRING, TRUE, FALSE, NULL + } + + } + + public static class PrestoDateConstant extends PrestoConstant { + + private final String textRepresentation; + + public PrestoDateConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepresentation); + } + + } + + public static class PrestoTimeConstant extends PrestoConstant { + + public final String textRepresentation; + + public PrestoTimeConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS"); + textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIME '%s'", textRepresentation); + } + + } + + public static class PrestoTimeWithTimeZoneConstant extends PrestoConstant { + + private final String textRepresentation; + private final String timeZone; + + public PrestoTimeWithTimeZoneConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("HH:mm:ss.SSS"); + textRepresentation = dateFormat.format(timestamp); + this.timeZone = Randomly.fromOptions(TIME_ZONES); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIME '%s %s'", textRepresentation, timeZone); + } + + } + + public static class PrestoTimestampConstant extends PrestoConstant { + + private final String textRepresentation; + + public PrestoTimestampConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + this.textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepresentation); + } + + } + + public static class PrestoTimestampWithTimezoneConstant extends PrestoConstant { + + private final String textRepresentation; + private final String timeZone; + + public PrestoTimestampWithTimezoneConstant(long val) { + Timestamp timestamp = new Timestamp(val); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + this.textRepresentation = dateFormat.format(timestamp); + this.timeZone = Randomly.fromOptions(TIME_ZONES); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s %s'", textRepresentation, timeZone); + } + + } + + public static class PrestoIntervalDayToSecondConstant extends PrestoConstant { + + private final String textRepresentation; + private final Interval fromInterval; + + public PrestoIntervalDayToSecondConstant() { + this.fromInterval = Randomly.fromOptions(Interval.values()); + SimpleDateFormat dateFormat = new SimpleDateFormat("dd HH:mm:ss"); + switch (fromInterval) { + case DAY: + dateFormat = new SimpleDateFormat("dd"); + break; + case HOUR: + dateFormat = new SimpleDateFormat("HH"); + break; + case MINUTE: + dateFormat = new SimpleDateFormat("mm"); + break; + case SECOND: + dateFormat = new SimpleDateFormat("ss"); + break; + default: + break; + } + + Randomly rand = new Randomly(); + + Timestamp timestamp = new Timestamp(rand.getLong(0, System.currentTimeMillis())); + this.textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + // if (toInterval == null) { + return String.format("INTERVAL '%s' %s", textRepresentation, fromInterval.name()); + // } else { + // return String.format("INTERVAL '%s' %s TO %s", textRepresentation, fromInterval, toInterval); + // } + } + + private enum Interval { + DAY, HOUR, MINUTE, SECOND + } + + } + + public static class PrestoIntervalYearToMonthConstant extends PrestoConstant { + + public String textRepresentation; + private final Interval fromInterval; + + public PrestoIntervalYearToMonthConstant() { + fromInterval = Randomly.fromOptions(Interval.values()); + SimpleDateFormat dateFormat; + switch (fromInterval) { + case YEAR: + dateFormat = new SimpleDateFormat("yyyy"); + break; + case MONTH: + dateFormat = new SimpleDateFormat("MM"); + break; + default: + dateFormat = new SimpleDateFormat("yyyy-MM"); + } + + Randomly rand = new Randomly(); + + Timestamp timestamp = new Timestamp(rand.getLong(0, System.currentTimeMillis())); + textRepresentation = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepresentation; + } + + @Override + public String toString() { + return String.format("INTERVAL '%s' %s", textRepresentation, fromInterval.name()); + } + + private enum Interval { + YEAR, MONTH + } + + } + + public static class PrestoBooleanConstant extends PrestoConstant { + + private final boolean value; + + public PrestoBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + @Override + public boolean asBoolean() { + return value; + } + + @Override + public boolean isBoolean() { + return true; + } + + } + + public static class PrestoArrayConstant extends PrestoConstant { + + private final List> elements; + + public PrestoArrayConstant(List> elements) { + this.elements = new ArrayList<>(elements); + } + + @Override + public boolean isArray() { + return true; + } + + @Override + public String toString() { + return "ARRAY[" + elements.stream().map(Object::toString).collect(Collectors.joining(", ")) + "]"; + } + + } + +} diff --git a/src/sqlancer/presto/ast/PrestoDateFunction.java b/src/sqlancer/presto/ast/PrestoDateFunction.java new file mode 100644 index 000000000..a8e00a3c0 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoDateFunction.java @@ -0,0 +1,523 @@ +package sqlancer.presto.ast; + +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; + +public enum PrestoDateFunction implements PrestoFunction { + + // Date and Time Functions# + // Returns the current date as of the start of the query. + CURRENT_DATE("current_date", PrestoDataType.DATE), + + // Returns the current time as of the start of the query. + CURRENT_TIME("current_time", PrestoDataType.TIME_WITH_TIME_ZONE), + + // Returns the current timestamp as of the start of the query. + CURRENT_TIMESTAMP("current_timestamp", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // Returns the current time zone in the format defined by IANA (e.g., America/Los_Angeles) or as fixed offset from + // UTC (e.g., +08:35) + CURRENT_TIMEZONE("current_timezone", PrestoDataType.VARCHAR), + + // This is an alias for CAST(x AS date). + DATE("date", PrestoDataType.DATE, PrestoDataType.DATE, PrestoDataType.INT, PrestoDataType.VARCHAR), + + // Returns the last day of the month. + LAST_DAY_OF_MONTH("last_day_of_month", PrestoDataType.DATE, PrestoDataType.DATE), + + // Parses the ISO 8601 formatted string into a timestamp with time zone. + FROM_ISO8601_TIMESTAMP("from_iso8601_timestamp", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.VARCHAR), + + // Parses the ISO 8601 formatted string into a date. + FROM_ISO8601_DATE("from_iso8601_date", PrestoDataType.DATE, PrestoDataType.VARCHAR), + + // Returns the UNIX timestamp unixtime as a timestamp. + FROM_UNIXTIME("from_unixtime", PrestoDataType.TIMESTAMP, PrestoDataType.INT), + + // Returns the UNIX timestamp unixtime as a timestamp with time zone using string for the time zone. + FROM_UNIXTIME_TIMEZONE("from_unixtime", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.INT, + PrestoDataType.VARCHAR) { + @Override + public boolean shouldPreserveOrderOfArguments() { + return true; + } + }, + + // Returns the UNIX timestamp unixtime as a timestamp with time zone using hours and minutes for the time zone + // offset. + FROM_UNIXTIME_HOURS_MINUTES("from_unixtime", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.INT, + PrestoDataType.INT) { + @Override + public boolean shouldPreserveOrderOfArguments() { + return true; + } + }, + + // Returns the current time as of the start of the query. -> time + LOCALTIME("localtime", PrestoDataType.TIME), + + // Returns the current timestamp as of the start of the query. -> timestamp + LOCALTIMESTAMP("localtimestamp", PrestoDataType.TIMESTAMP), + + // This is an alias for current_timestamp. → timestamp with time zone# + NOW("now", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // Formats x as an ISO 8601 string. x can be date, timestamp, or timestamp with time zone. → varchar# + TO_ISO8601("to_iso8601", PrestoDataType.VARCHAR, PrestoDataType.DATE, PrestoDataType.TIMESTAMP, + PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // Returns the day-to-second interval as milliseconds. → bigint# + TO_MILLISECONDS("to_milliseconds", PrestoDataType.INT, PrestoDataType.INTERVAL_DAY_TO_SECOND), + TO_MILLISECONDS_2("to_milliseconds", PrestoDataType.INT, PrestoDataType.INTERVAL_YEAR_TO_MONTH), + + // Returns timestamp as a UNIX timestamp. → double# + TO_UNIXTIME("to_unixtime", PrestoDataType.FLOAT, PrestoDataType.TIMESTAMP), + TO_UNIXTIME_2("to_unixtime", PrestoDataType.FLOAT, PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + + // The following SQL-standard functions do not use parenthesis: + CURRENT_DATE_NA("current_date", PrestoDataType.DATE) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + CURRENT_TIME_NA("current_time", PrestoDataType.TIME) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + CURRENT_TIMESTAMP_NA("current_timestamp", PrestoDataType.TIMESTAMP) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + LOCALTIME_NA("localtime", PrestoDataType.TIME) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + LOCALTIMESTAMP_NA("localtimestamp", PrestoDataType.TIMESTAMP) { + @Override + public boolean isStandardFunction() { + return false; + } + }, + + // Truncation Function + // date_trunc(unit, x) → [same as input] + DATE_TRUNC_1("date_trunc", PrestoDataType.TIMESTAMP, PrestoDataType.VARCHAR, PrestoDataType.TIMESTAMP), + DATE_TRUNC_2("date_trunc", PrestoDataType.TIMESTAMP_WITH_TIME_ZONE, PrestoDataType.VARCHAR, + PrestoDataType.TIMESTAMP_WITH_TIME_ZONE), + DATE_TRUNC_3("date_trunc", PrestoDataType.DATE, PrestoDataType.VARCHAR, PrestoDataType.DATE), + DATE_TRUNC_4("date_trunc", PrestoDataType.TIME, PrestoDataType.VARCHAR, PrestoDataType.TIME); + + /* + * + * Interval Functions# The functions in this section support the following interval units: + * + * Unit + * + * Description + * + * millisecond + * + * Milliseconds + * + * second + * + * Seconds + * + * minute + * + * Minutes + * + * hour + * + * Hours + * + * day + * + * Days + * + * week + * + * Weeks + * + * month + * + * Months + * + * quarter + * + * Quarters of a year + * + * year + * + * Years + * + * date_add(unit, value, timestamp) → [same as input]# Adds an interval value of type unit to timestamp. Subtraction + * can be performed by using a negative value. + * + * date_diff(unit, timestamp1, timestamp2) → bigint# Returns timestamp2 - timestamp1 expressed in terms of unit. + * + * Duration Function# The parse_duration function supports the following units: + * + * Unit + * + * Description + * + * ns + * + * Nanoseconds + * + * us + * + * Microseconds + * + * ms + * + * Milliseconds + * + * s + * + * Seconds + * + * m + * + * Minutes + * + * h + * + * Hours + * + * d + * + * Days + * + * parse_duration(string) → interval# Parses string of format value unit into an interval, where value is fractional + * number of unit values: + * + * SELECT parse_duration('42.8ms'); -- 0 00:00:00.043 SELECT parse_duration('3.81 d'); -- 3 19:26:24.000 SELECT + * parse_duration('5m'); -- 0 00:05:00.000 MySQL Date Functions# The functions in this section use a format string + * that is compatible with the MySQL date_parse and str_to_date functions. The following table, based on the MySQL + * manual, describes the format specifiers: + * + * Specifier + * + * Description + * + * %a + * + * Abbreviated weekday name (Sun .. Sat) + * + * %b + * + * Abbreviated month name (Jan .. Dec) + * + * %c + * + * Month, numeric (1 .. 12) 4 + * + * %D + * + * Day of the month with English suffix (0th, 1st, 2nd, 3rd, …) + * + * %d + * + * Day of the month, numeric (01 .. 31) 4 + * + * %e + * + * Day of the month, numeric (1 .. 31) 4 + * + * %f + * + * Fraction of second (6 digits for printing: 000000 .. 999000; 1 - 9 digits for parsing: 0 .. 999999999) 1 + * + * %H + * + * Hour (00 .. 23) + * + * %h + * + * Hour (01 .. 12) + * + * %I + * + * Hour (01 .. 12) + * + * %i + * + * Minutes, numeric (00 .. 59) + * + * %j + * + * Day of year (001 .. 366) + * + * %k + * + * Hour (0 .. 23) + * + * %l + * + * Hour (1 .. 12) + * + * %M + * + * Month name (January .. December) + * + * %m + * + * Month, numeric (01 .. 12) 4 + * + * %p + * + * AM or PM + * + * %r + * + * Time, 12-hour (hh:mm:ss followed by AM or PM) + * + * %S + * + * Seconds (00 .. 59) + * + * %s + * + * Seconds (00 .. 59) + * + * %T + * + * Time, 24-hour (hh:mm:ss) + * + * %U + * + * Week (00 .. 53), where Sunday is the first day of the week + * + * %u + * + * Week (00 .. 53), where Monday is the first day of the week + * + * %V + * + * Week (01 .. 53), where Sunday is the first day of the week; used with %X + * + * %v + * + * Week (01 .. 53), where Monday is the first day of the week; used with %x + * + * %W + * + * Weekday name (Sunday .. Saturday) + * + * %w + * + * Day of the week (0 .. 6), where Sunday is the first day of the week 3 + * + * %X + * + * Year for the week where Sunday is the first day of the week, numeric, four digits; used with %V + * + * %x + * + * Year for the week, where Monday is the first day of the week, numeric, four digits; used with %v + * + * %Y + * + * Year, numeric, four digits + * + * %y + * + * Year, numeric (two digits) 2 + * + * %% + * + * A literal % character + * + * %x + * + * x, for any x not listed above + * + * 1 Timestamp is truncated to milliseconds. + * + * 2 When parsing, two-digit year format assumes range 1970 ... 2069, so “70” will result in year 1970 but “69” will + * produce 2069. + * + * 3 This specifier is not supported yet. Consider using day_of_week() (it uses 1-7 instead of 0-6). + * + * 4(1,2,3,4) This specifier does not support 0 as a month or day. + * + * Warning + * + * The following specifiers are not currently supported: %D %U %u %V %w %X + * + * date_format(timestamp, format) → varchar# Formats timestamp as a string using format. + * + * date_parse(string, format) → timestamp# Parses string into a timestamp using format. + * + * Java Date Functions# The functions in this section use a format string that is compatible with JodaTime’s + * DateTimeFormat pattern format. + * + * format_datetime(timestamp, format) → varchar# Formats timestamp as a string using format. + * + * parse_datetime(string, format) → timestamp with time zone# Parses string into a timestamp with time zone using + * format. + * + * Extraction Function# The extract function supports the following fields: + * + * Field + * + * Description + * + * YEAR + * + * year() + * + * QUARTER + * + * quarter() + * + * MONTH + * + * month() + * + * WEEK + * + * week() + * + * DAY + * + * day() + * + * DAY_OF_MONTH + * + * day() + * + * DAY_OF_WEEK + * + * day_of_week() + * + * DOW + * + * day_of_week() + * + * DAY_OF_YEAR + * + * day_of_year() + * + * DOY + * + * day_of_year() + * + * YEAR_OF_WEEK + * + * year_of_week() + * + * YOW + * + * year_of_week() + * + * HOUR + * + * hour() + * + * MINUTE + * + * minute() + * + * SECOND + * + * second() + * + * TIMEZONE_HOUR + * + * timezone_hour() + * + * TIMEZONE_MINUTE + * + * timezone_minute() + * + * The types supported by the extract function vary depending on the field to be extracted. Most fields support all + * date and time types. + * + * extract(field FROM x) → bigint# Returns field from x. + * + * Note + * + * This SQL-standard function uses special syntax for specifying the arguments. + * + * Convenience Extraction Functions# day(x) → bigint# Returns the day of the month from x. + * + * day_of_month(x) → bigint# This is an alias for day(). + * + * day_of_week(x) → bigint# Returns the ISO day of the week from x. The value ranges from 1 (Monday) to 7 (Sunday). + * + * day_of_year(x) → bigint# Returns the day of the year from x. The value ranges from 1 to 366. + * + * dow(x) → bigint# This is an alias for day_of_week(). + * + * doy(x) → bigint# This is an alias for day_of_year(). + * + * hour(x) → bigint# Returns the hour of the day from x. The value ranges from 0 to 23. + * + * millisecond(x) → bigint# Returns the millisecond of the second from x. + * + * minute(x) → bigint# Returns the minute of the hour from x. + * + * month(x) → bigint# Returns the month of the year from x. + * + * quarter(x) → bigint# Returns the quarter of the year from x. The value ranges from 1 to 4. + * + * second(x) → bigint# Returns the second of the minute from x. + * + * timezone_hour(timestamp) → bigint# Returns the hour of the time zone offset from timestamp. + * + * timezone_minute(timestamp) → bigint# Returns the minute of the time zone offset from timestamp. + * + * week(x) → bigint# Returns the ISO week of the year from x. The value ranges from 1 to 53. + * + * week_of_year(x) → bigint# This is an alias for week(). + * + * year(x) → bigint# Returns the year from x. + * + * year_of_week(x) → bigint# Returns the year of the ISO week from x. + * + * yow(x) → bigint# This is an alias for year_of_week(). + * + * + * + */ + + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoDateFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoDefaultFunction.java b/src/sqlancer/presto/ast/PrestoDefaultFunction.java new file mode 100644 index 000000000..95b89091e --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoDefaultFunction.java @@ -0,0 +1,233 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public enum PrestoDefaultFunction implements PrestoFunction { + + // Conditional functions + IF_TRUE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { PrestoSchema.PrestoDataType.BOOLEAN, + returnType.getPrimitiveDataType() }; + } + }, + + IF_TRUE_FALSE("if", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { PrestoSchema.PrestoDataType.BOOLEAN, + returnType.getPrimitiveDataType(), returnType.getPrimitiveDataType() }; + } + }, + + NULLIF("nullif", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return new PrestoDataType[] { returnType.getPrimitiveDataType(), returnType.getPrimitiveDataType() }; + } + }, + + COALESCE("coalesce", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return true; + } + + @Override + public int getNumberOfArguments() { + return UNLIMITED_NO_OF_ARGUMENTS; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + ArrayList prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { + return super.getArgumentsForReturnType(gen, depth, argumentTypes, returnType); + } + }, + + // comparison + + // Returns the largest of the provided values. → [same as input] + GREATEST("greatest", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return UNLIMITED_NO_OF_ARGUMENTS; + } + + @Override + public PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType) { + return new PrestoSchema.PrestoDataType[] { returnType.getPrimitiveDataType() }; + } + }, + // Returns the smallest of the provided values. → [same as input] + LEAST("least", null) { + @Override + public boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType) { + return PrestoDataType.getOrderableTypes().contains(returnType.getPrimitiveDataType()); + } + + @Override + public int getNumberOfArguments() { + return UNLIMITED_NO_OF_ARGUMENTS; + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + ArrayList prestoDataTypes = new ArrayList<>(); + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + prestoDataTypes.add(returnType.getPrimitiveDataType()); + } + return prestoDataTypes.toArray(new PrestoDataType[0]); + } + }; + + private static final int UNLIMITED_NO_OF_ARGUMENTS = -1; + private final PrestoDataType returnType; + private final PrestoDataType[] argumentTypes; + private final String functionName; + + PrestoDefaultFunction(String functionName, PrestoDataType returnType) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = new PrestoDataType[0]; + } + + PrestoDefaultFunction(PrestoDataType returnType) { + this.returnType = returnType; + this.argumentTypes = new PrestoDataType[0]; + this.functionName = toString(); + } + + PrestoDefaultFunction(PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + this.functionName = toString(); + } + + PrestoDefaultFunction(String functionName, PrestoDataType returnType, PrestoDataType... argumentTypes) { + this.functionName = functionName; + this.returnType = returnType; + this.argumentTypes = argumentTypes.clone(); + } + + public static List getFunctionsCompatibleWith(PrestoCompositeDataType returnType) { + return Stream.of(values()).filter(f -> f.isCompatibleWithReturnType(returnType)).collect(Collectors.toList()); + } + + @Override + public String getFunctionName() { + return functionName; + } + + @Override + public int getNumberOfArguments() { + return argumentTypes == null ? 0 : argumentTypes.length; + } + + @Override + public boolean isCompatibleWithReturnType(PrestoCompositeDataType returnType) { + return this.returnType == returnType.getPrimitiveDataType(); + } + + @Override + public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { + return argumentTypes.clone(); + } + + @Override + public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { + List> arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to the same element type. + PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + + if (getNumberOfArguments() == UNLIMITED_NO_OF_ARGUMENTS) { + PrestoDataType dataType = getArgumentTypes(returnType)[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoCompositeDataType type; + + if (dataType == PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + type = savedArrayType; + } else { + type = PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(gen.generateExpression(type, depth + 1)); + } + } else { + for (PrestoDataType arg : argumentTypes) { + PrestoCompositeDataType type; + if (arg == PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = arg.get(); + } + type = savedArrayType; + } else { + type = PrestoCompositeDataType.fromDataType(arg); + } + arguments.add(gen.generateExpression(type, depth + 1)); + + } + } + return arguments; + } + + @Override + public String toString() { + if (functionName != null) { + return functionName; + } + return super.toString(); + } + +} diff --git a/src/sqlancer/presto/ast/PrestoExpression.java b/src/sqlancer/presto/ast/PrestoExpression.java new file mode 100644 index 000000000..143eb6823 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoExpression.java @@ -0,0 +1,5 @@ +package sqlancer.presto.ast; + +public interface PrestoExpression { + +} diff --git a/src/sqlancer/presto/ast/PrestoFunction.java b/src/sqlancer/presto/ast/PrestoFunction.java new file mode 100644 index 000000000..c91759e2f --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoFunction.java @@ -0,0 +1,127 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public interface PrestoFunction { + + String getFunctionName(); + + boolean isCompatibleWithReturnType(PrestoSchema.PrestoCompositeDataType returnType); + + PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType); + + default List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoSchema.PrestoDataType[] argumentTypes, PrestoSchema.PrestoCompositeDataType returnType) { + + List> arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to + // the same element type. + PrestoSchema.PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + // -1 - unlimited number of arguments + if (getNumberOfArguments() == -1) { + PrestoSchema.PrestoDataType dataType = argumentTypes[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoSchema.PrestoCompositeDataType type; + + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + type = savedArrayType; + } else { + type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(gen.generateExpression(type, depth + 1)); + } + } else { + for (PrestoSchema.PrestoDataType arg : argumentTypes) { + PrestoSchema.PrestoCompositeDataType dataType; + if (arg == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = arg.get(); + } + dataType = savedArrayType; + } else { + dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); + } + Node expression = gen.generateExpression(dataType, depth + 1); + arguments.add(expression); + } + } + return arguments; + } + + default List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + PrestoSchema.PrestoCompositeDataType returnType, boolean orderable) { + + List> arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to + // the same element type. + PrestoSchema.PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + if (getNumberOfArguments() == -1) { + PrestoSchema.PrestoDataType dataType = getArgumentTypes(returnType)[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoSchema.PrestoCompositeDataType compositeDataType; + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + compositeDataType = savedArrayType; + } else { + compositeDataType = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(gen.generateExpression(compositeDataType, depth + 1)); + } + } else { + for (PrestoSchema.PrestoDataType dataType : getArgumentTypes(returnType)) { + PrestoSchema.PrestoCompositeDataType compositeDataType; + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + PrestoSchema.PrestoCompositeDataType arrayType; + do { + arrayType = dataType.get(); + } while (!arrayType.getElementType().isOrderable()); + savedArrayType = arrayType; + } + compositeDataType = savedArrayType; + } else { + compositeDataType = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + Node expression = gen.generateExpression(compositeDataType, depth + 1); + arguments.add(expression); + } + } + return arguments; + } + + default int getNumberOfArguments() { + return getArgumentTypes(null).length; + } + + default boolean shouldPreserveOrderOfArguments() { + return false; + } + + default boolean isStandardFunction() { + return true; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java new file mode 100644 index 000000000..5b184671b --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java @@ -0,0 +1,24 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; + +public class PrestoFunctionWithoutParenthesis implements Node { + + private final PrestoSchema.PrestoCompositeDataType type; + private final String expr; + + public PrestoFunctionWithoutParenthesis(String expr, PrestoSchema.PrestoCompositeDataType type) { + this.expr = expr; + this.type = type; + } + + public String getExpr() { + return expr; + } + + public PrestoSchema.PrestoCompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoJoin.java b/src/sqlancer/presto/ast/PrestoJoin.java new file mode 100644 index 000000000..79bf7e1fc --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoJoin.java @@ -0,0 +1,118 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public class PrestoJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + private OuterType outerType; + + public PrestoJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public static List> getJoins( + List> tableList, PrestoGlobalState globalState) { + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + PrestoTypedExpressionGenerator joinGen = new PrestoTypedExpressionGenerator(globalState) + .setColumns(columns); + switch (JoinType.getRandom()) { + case INNER: + joinExpressions.add(PrestoJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN)))); + break; + case LEFT: + joinExpressions.add(PrestoJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN)))); + break; + case RIGHT: + joinExpressions.add(PrestoJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN)))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static PrestoJoin createRightOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new PrestoJoin(left, right, JoinType.RIGHT, predicate); + } + + public static PrestoJoin createLeftOuterJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new PrestoJoin(left, right, JoinType.LEFT, predicate); + } + + public static PrestoJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new PrestoJoin(left, right, JoinType.INNER, predicate); + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + public OuterType getOuterType() { + return outerType; + } + + @SuppressWarnings("unused") + private void setOuterType(OuterType outerType) { + this.outerType = outerType; + } + + public enum JoinType { + INNER, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + + public enum OuterType { + FULL, LEFT, RIGHT; + + public static OuterType getRandom() { + return Randomly.fromOptions(values()); + } + } + +} diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java new file mode 100644 index 000000000..47fc209fb --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java @@ -0,0 +1,39 @@ +package sqlancer.presto.ast; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.ast.newast.Node; + +public class PrestoMultiValuedComparison implements Node { + + private final Node left; + private final List> right; + private final PrestoMultiValuedComparisonType type; + private final PrestoMultiValuedComparisonOperator op; + + public PrestoMultiValuedComparison(Node left, List> right, + PrestoMultiValuedComparisonType type, PrestoMultiValuedComparisonOperator op) { + this.left = left; + this.right = new ArrayList<>(right); + this.type = type; + this.op = op; + } + + public Node getLeft() { + return left; + } + + public PrestoMultiValuedComparisonOperator getOp() { + return op; + } + + public List> getRight() { + return new ArrayList<>(right); + } + + public PrestoMultiValuedComparisonType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java new file mode 100644 index 000000000..829f554f4 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonOperator.java @@ -0,0 +1,43 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; +import sqlancer.presto.PrestoSchema; + +public enum PrestoMultiValuedComparisonOperator { + EQUALS("="), NOT_EQUALS("<>"), NOT_EQUALS_ALT("!="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), + SMALLER_EQUALS("<="); + + private final String stringRepresentation; + + PrestoMultiValuedComparisonOperator(String stringRepresentation) { + this.stringRepresentation = stringRepresentation; + } + + public static PrestoMultiValuedComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + public static PrestoMultiValuedComparisonOperator getRandomForType(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoDataType dataType = type.getPrimitiveDataType(); + + switch (dataType) { + case BOOLEAN: + case INT: + case FLOAT: + case DECIMAL: + case DATE: + case TIME: + case TIMESTAMP: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return getRandom(); + default: + return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT); + } + } + + public String getStringRepresentation() { + return stringRepresentation; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java new file mode 100644 index 000000000..d1ecdd382 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparisonType.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; + +public enum PrestoMultiValuedComparisonType { + ANY, SOME, ALL; + + public static PrestoMultiValuedComparisonType getRandom() { + return Randomly.fromOptions(values()); + } +} diff --git a/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java new file mode 100644 index 000000000..d51539645 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java @@ -0,0 +1,36 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.Node; + +public class PrestoQuantifiedComparison implements Node { + + private final Node left; + private final PrestoSelect right; + private final PrestoMultiValuedComparisonType type; + private final PrestoMultiValuedComparisonOperator op; + + public PrestoQuantifiedComparison(Node left, PrestoSelect right, + PrestoMultiValuedComparisonType type, PrestoMultiValuedComparisonOperator op) { + this.left = left; + this.right = right; + this.type = type; + this.op = op; + } + + public Node getLeft() { + return left; + } + + public PrestoMultiValuedComparisonOperator getOp() { + return op; + } + + public Node getRight() { + return right; + } + + public PrestoMultiValuedComparisonType getType() { + return type; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoSelect.java b/src/sqlancer/presto/ast/PrestoSelect.java new file mode 100644 index 000000000..c99c5c7d0 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoSelect.java @@ -0,0 +1,18 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Node; + +public class PrestoSelect extends SelectBase> implements Node { + + private boolean isDistinct; + + public boolean isDistinct() { + return isDistinct; + } + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + +} diff --git a/src/sqlancer/presto/ast/PrestoTableReference.java b/src/sqlancer/presto/ast/PrestoTableReference.java new file mode 100644 index 000000000..4c0e0d789 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.presto.PrestoSchema; + +public class PrestoTableReference extends TableReferenceNode { + + public PrestoTableReference(PrestoSchema.PrestoTable table) { + super(table); + } +} diff --git a/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java new file mode 100644 index 000000000..2bdc142e6 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java @@ -0,0 +1,52 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; + +public class PrestoUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { + + public PrestoUnaryPostfixOperation(Node expr, PrestoUnaryPostfixOperator op) { + super(expr, op); + } + + public Node getExpression() { + return getExpr(); + } + + public enum PrestoUnaryPostfixOperator implements BinaryOperatorNode.Operator { + IS_NULL("IS NULL") { + @Override + public PrestoSchema.PrestoDataType[] getInputDataTypes() { + return PrestoSchema.PrestoDataType.values(); + } + }, + IS_NOT_NULL("IS NOT NULL") { + @Override + public PrestoSchema.PrestoDataType[] getInputDataTypes() { + return PrestoSchema.PrestoDataType.values(); + } + }; + + private final String textRepresentations; + + PrestoUnaryPostfixOperator(String text) { + this.textRepresentations = text; + } + + public static PrestoUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentations; + } + + public abstract PrestoSchema.PrestoDataType[] getInputDataTypes(); + + } + +} diff --git a/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java new file mode 100644 index 000000000..cb8313622 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java @@ -0,0 +1,62 @@ +package sqlancer.presto.ast; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoSchema; + +public class PrestoUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { + + public PrestoUnaryPrefixOperation(PrestoUnaryPrefixOperator operation, Node expression) { + super(expression, operation); + } + + public enum PrestoUnaryPrefixOperator implements BinaryOperatorNode.Operator { + NOT("NOT", PrestoSchema.PrestoDataType.BOOLEAN) { + @Override + public PrestoSchema.PrestoDataType getExpressionType() { + return PrestoSchema.PrestoDataType.BOOLEAN; + } + }, + + UNARY_PLUS("+", PrestoSchema.PrestoDataType.INT, PrestoSchema.PrestoDataType.FLOAT, + PrestoSchema.PrestoDataType.DECIMAL) { + @Override + public PrestoSchema.PrestoDataType getExpressionType() { + return PrestoSchema.PrestoDataType.INT; + } + }, + UNARY_MINUS("-", PrestoSchema.PrestoDataType.INT, PrestoSchema.PrestoDataType.FLOAT, + PrestoSchema.PrestoDataType.DECIMAL) { + @Override + public PrestoSchema.PrestoDataType getExpressionType() { + return PrestoSchema.PrestoDataType.INT; + } + }; + + private final String textRepresentation; + private final PrestoSchema.PrestoDataType[] dataTypes; + + PrestoUnaryPrefixOperator(String textRepresentation, PrestoSchema.PrestoDataType... dataTypes) { + this.textRepresentation = textRepresentation; + this.dataTypes = dataTypes.clone(); + } + + public PrestoSchema.PrestoDataType getRandomInputDataTypes() { + return Randomly.fromOptions(dataTypes); + } + + public abstract PrestoSchema.PrestoDataType getExpressionType(); + + @Override + public String getTextRepresentation() { + return this.textRepresentation; + } + + public PrestoSchema.PrestoDataType getExpressionType(PrestoSchema.PrestoDataType type) { + return type; + } + } + +} diff --git a/src/sqlancer/presto/gen/PrestoAlterTableGenerator.java b/src/sqlancer/presto/gen/PrestoAlterTableGenerator.java new file mode 100644 index 000000000..f3d5f9d55 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoAlterTableGenerator.java @@ -0,0 +1,67 @@ +package sqlancer.presto.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoTable; + +public final class PrestoAlterTableGenerator { + + private PrestoAlterTableGenerator() { + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder("ALTER TABLE "); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + // PrestoTypedExpressionGenerator gen = new + // PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append(table.getName()); + sb.append(" "); + Action action = Randomly.fromOptions(Action.values()); + switch (action) { + case ADD_COLUMN: + sb.append("ADD COLUMN "); + String columnName = table.getFreeColumnName(); + sb.append(columnName); + sb.append(" "); + sb.append(PrestoCompositeDataType.getRandomWithoutNull()); + break; + case ALTER_COLUMN: + sb.append("ALTER COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" SET DATA TYPE "); + sb.append(PrestoCompositeDataType.getRandomWithoutNull()); + // if (Randomly.getBoolean()) { + // sb.append(" USING "); + // PrestoErrors.addExpressionErrors(errors); + // sb.append(PrestoToStringVisitor.asString(gen.generateExpression())); + // } + errors.add("Cannot change the type of this column: an index depends on it!"); + errors.add("Cannot change the type of a column that has a UNIQUE or PRIMARY KEY constraint specified"); + errors.add("Unimplemented type for cast"); + errors.add("Conversion:"); + errors.add("Cannot change the type of a column that has a CHECK constraint specified"); + break; + case DROP_COLUMN: + sb.append("DROP COLUMN "); + sb.append(table.getRandomColumn().getName()); + errors.add("named in key does not exist"); // TODO + errors.add("Cannot drop this column:"); + errors.add("Cannot drop column: table only has one column remaining!"); + errors.add("because there is a CHECK constraint that depends on it"); + errors.add("because there is a UNIQUE constraint that depends on it"); + break; + default: + throw new AssertionError(action); + } + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + + enum Action { + ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN + } + +} diff --git a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java new file mode 100644 index 000000000..9f869c241 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java @@ -0,0 +1,32 @@ +package sqlancer.presto.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; + +public final class PrestoDeleteGenerator { + + private PrestoDeleteGenerator() { + } + + public static SQLQueryAdapter generate(PrestoGlobalState globalState) { + StringBuilder sb = new StringBuilder("DELETE FROM "); + ExpectedErrors errors = new ExpectedErrors(); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + sb.append(PrestoToStringVisitor + .asString(new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()))); + } + PrestoErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java new file mode 100644 index 000000000..7a467e37d --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -0,0 +1,57 @@ +package sqlancer.presto.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public final class PrestoIndexGenerator { + + private PrestoIndexGenerator() { + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE "); + if (Randomly.getBoolean()) { + errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + sb.append("UNIQUE "); + } + sb.append("INDEX "); + sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information + sb.append(" ON "); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + sb.append("("); + List columns = table.getRandomNonEmptyColumnSubset(); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + sb.append(" "); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + } + sb.append(")"); + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + Node expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) + .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + sb.append(PrestoToStringVisitor.asString(expr)); + } + errors.add("already exists!"); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java new file mode 100644 index 000000000..102d2f1be --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -0,0 +1,52 @@ +package sqlancer.presto.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoInsertGenerator extends AbstractInsertGenerator { + + private final PrestoGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public PrestoInsertGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + return new PrestoInsertGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + sb.append(table.getName()); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); + PrestoErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } + + @Override + protected void insertValue(PrestoColumn prestoColumn) { + Node constant = new PrestoTypedExpressionGenerator(globalState) + .generateInsertConstant(prestoColumn.getType()); + sb.append(PrestoToStringVisitor.asString(constant)); + + } + +} diff --git a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java new file mode 100644 index 000000000..167c1f222 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java @@ -0,0 +1,72 @@ +package sqlancer.presto.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.ast.PrestoConstant; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; + +public final class PrestoRandomQuerySynthesizer { + + private PrestoRandomQuerySynthesizer() { + } + + public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrColumns) { + PrestoTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); + PrestoSelect select = new PrestoSelect(); + // TODO: distinct + // select.setDistinct(Randomly.getBoolean()); + // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); + List> columns = new ArrayList<>(); + for (int i = 0; i < nrColumns; i++) { + // if (allowAggregates && Randomly.getBoolean()) { + Node expression = gen + .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + columns.add(expression); + // } else { + // columns.add(gen()); + // } + } + select.setFetchColumns(columns); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = PrestoJoin.getJoins(tableList, globalState); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableList)); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull())); + } + if (Randomly.getBoolean()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + + if (Randomly.getBoolean()) { + select.setLimitClause(PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + } + // if (Randomly.getBoolean()) { + // select.setOffsetClause( + // PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + // } + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + return select; + } + +} diff --git a/src/sqlancer/presto/gen/PrestoTableGenerator.java b/src/sqlancer/presto/gen/PrestoTableGenerator.java new file mode 100644 index 000000000..1d7df2ee6 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoTableGenerator.java @@ -0,0 +1,69 @@ +package sqlancer.presto.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; + +public class PrestoTableGenerator { + + private static List getNewColumns() { + List columns = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + String columnName = String.format("c%d", i); + PrestoCompositeDataType columnType = PrestoCompositeDataType.getRandomWithoutNull(); + columns.add(new PrestoColumn(columnName, columnType, false, false)); + } + return columns; + } + + public SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + String catalog = globalState.getDbmsSpecificOptions().catalog; + String schema = globalState.getDatabaseName(); + + sb.append(catalog).append("."); + sb.append(schema).append("."); + + sb.append(tableName); + sb.append("("); + List columns = getNewColumns(); + // TypedExpressionGenerator, PrestoColumn, PrestoCompositeDataType> + // typedExpressionGenerator = new PrestoTypedExpressionGenerator(globalState).setColumns(columns); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + PrestoColumn column = columns.get(i); + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + // sb.append(" UNIQUE"); + // } + // if (globalState.getDbmsSpecificOptions().testNotNullConstraints + // && Randomly.getBooleanWithRatherLowProbability()) { + // sb.append(" NOT NULL"); + // } + } + // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { + // errors.add("Invalid type for index"); + // List primaryKeyColumns = Randomly.nonEmptySubset(columns); + // sb.append(", PRIMARY KEY("); + // sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + // sb.append(")"); + // } + sb.append(")"); + + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java new file mode 100644 index 000000000..7afb8da29 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -0,0 +1,798 @@ +package sqlancer.presto.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBetweenOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewCaseOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.common.ast.newast.NewTernaryNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.ast.PrestoAggregateFunction; +import sqlancer.presto.ast.PrestoAtTimeZoneOperator; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoColumnReference; +import sqlancer.presto.ast.PrestoConstant; +import sqlancer.presto.ast.PrestoDefaultFunction; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoMultiValuedComparison; +import sqlancer.presto.ast.PrestoMultiValuedComparisonOperator; +import sqlancer.presto.ast.PrestoMultiValuedComparisonType; +import sqlancer.presto.ast.PrestoQuantifiedComparison; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoUnaryPostfixOperation; +import sqlancer.presto.ast.PrestoUnaryPrefixOperation; + +public final class PrestoTypedExpressionGenerator extends + TypedExpressionGenerator, PrestoSchema.PrestoColumn, PrestoSchema.PrestoCompositeDataType> { + + private final Randomly randomly; + private final PrestoGlobalState globalState; + private final int maxDepth; + + public PrestoTypedExpressionGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.randomly = globalState.getRandomly(); + this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); + } + + @Override + public Node generatePredicate() { + return generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), + randomly.getInteger(0, maxDepth)); + } + + @Override + public Node negatePredicate(Node predicate) { + return new PrestoUnaryPrefixOperation(PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT, predicate); + } + + @Override + public Node isNull(Node expr) { + return new PrestoUnaryPostfixOperation(expr, PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); + } + + @Override + public Node generateConstant(PrestoSchema.PrestoCompositeDataType type) { + if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { + return PrestoConstant.createArrayConstant(type); + // case MAP: + // return PrestoConstant.createMapConstant(type); + } + return PrestoConstant.generateConstant(type, false); + } + + public Node generateInsertConstant(PrestoSchema.PrestoCompositeDataType type) { + if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { + return PrestoConstant.createArrayConstant(type); + // case MAP: + // return PrestoConstant.createMapConstant(type); + } + return PrestoConstant.generateConstant(type, true); + } + + @Override + public Node generateExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (allowAggregates && Randomly.getBoolean()) { + return generateAggregate(type); + } + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } else { + // TODO: functions + List applicableFunctions = PrestoDefaultFunction.getFunctionsCompatibleWith(type); + if (Randomly.getBooleanWithRatherLowProbability() && !applicableFunctions.isEmpty()) { + PrestoDefaultFunction function = Randomly.fromList(applicableFunctions); + return generateFunction(type, depth, function); + } + // TODO: try + // if (Randomly.getBooleanWithRatherLowProbability()) { + // return generateTry(type, depth); + // } + + // TODO: cast + // + // if (Randomly.getBooleanWithRatherLowProbability()) { + // Node expressionNode = generateCast(type, depth); + // } + if (Randomly.getBooleanWithRatherLowProbability()) { + return getCase(type, depth); + } + switch (type.getPrimitiveDataType()) { + case BOOLEAN: + return generateBooleanExpression(depth); + case VARCHAR: + case CHAR: + return generateStringExpression(type, depth); + case INT: + case DECIMAL: + case FLOAT: + return generateNumericExpression(depth); + case DATE: + case TIME: + case TIMESTAMP: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return generateTemporalExpression(type, depth); + case INTERVAL_YEAR_TO_MONTH: + case INTERVAL_DAY_TO_SECOND: + return generateIntervalExpression(type, depth); + case JSON: + return generateJsonExpression(type); + case VARBINARY: + case ARRAY: + // case MAP: + return generateLeafNode(type); // TODO + default: + throw new AssertionError(type); + } + } + } + + private Node generateJsonExpression(PrestoSchema.PrestoCompositeDataType type) { + return generateLeafNode(type); + } + + private Node generateCast(PrestoSchema.PrestoCompositeDataType type, int depth) { + // check can cast + Node expressionNode = generateExpression(getRandomType(), depth + 1); + return new PrestoCastFunction(expressionNode, type); + } + + @SuppressWarnings("unused") + private Node generateTry(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (type.getPrimitiveDataType().isNumeric() && Randomly.getBooleanWithRatherLowProbability()) { + Node expression = generateExpression(type); + return new NewFunctionNode<>(List.of(expression), "try"); + } + + List applicableFunctions = PrestoDefaultFunction.getFunctionsCompatibleWith(type); + if (Randomly.getBooleanWithRatherLowProbability() && !applicableFunctions.isEmpty()) { + PrestoDefaultFunction function = Randomly.fromList(applicableFunctions); + Node expression = generateFunction(type, depth, function); + return new NewFunctionNode<>(List.of(expression), "try"); + } + return new NewFunctionNode<>(List.of(generateCast(type, depth)), "try"); + } + + private NewCaseOperatorNode getCase(PrestoSchema.PrestoCompositeDataType type, int depth) { + List> conditions = new ArrayList<>(); + List> cases = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + conditions.add(generateExpression(type, depth + 1)); + cases.add(generateExpression(type, depth + 1)); + } + Node elseExpr = null; + if (Randomly.getBoolean()) { + elseExpr = generateExpression(type, depth + 1); + } + Node expression = generateExpression(type); + return new NewCaseOperatorNode<>(expression, conditions, cases, elseExpr); + } + + private Node generateFunction(PrestoSchema.PrestoCompositeDataType returnType, int depth, + PrestoDefaultFunction function) { + + PrestoSchema.PrestoDataType[] argumentTypes = function.getArgumentTypes(returnType); + List> arguments = new ArrayList<>(); + + // This is a workaround based on the assumption that array types should refer to + // the same element type. + PrestoSchema.PrestoCompositeDataType savedArrayType = null; + if (returnType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { + savedArrayType = returnType; + } + if (function.getNumberOfArguments() == -1) { + PrestoSchema.PrestoDataType dataType = argumentTypes[0]; + // TODO: consider upper + long no = Randomly.getNotCachedInteger(2, 10); + for (int i = 0; i < no; i++) { + PrestoSchema.PrestoCompositeDataType type; + + if (dataType == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = dataType.get(); + } + type = savedArrayType; + } else { + type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + } + arguments.add(generateExpression(type, depth + 1)); + } + } else { + for (PrestoSchema.PrestoDataType arg : argumentTypes) { + PrestoSchema.PrestoCompositeDataType dataType; + if (arg == PrestoSchema.PrestoDataType.ARRAY) { + if (savedArrayType == null) { + savedArrayType = arg.get(); + } + dataType = savedArrayType; + } else { + dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); + } + Node expression = generateExpression(dataType, depth + 1); + arguments.add(expression); + } + } + return new NewFunctionNode<>(arguments, function); + } + + private Node generateStringExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(type); + } + return getStringOperation(depth); + } + + private NewBinaryOperatorNode getStringOperation(int depth) { + StringExpression exprType = Randomly.fromOptions(StringExpression.values()); + if (Objects.requireNonNull(exprType) == StringExpression.CONCAT) { + Node left = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + Node right = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + PrestBinaryStringOperator operator = PrestBinaryStringOperator.CONCAT; + return new NewBinaryOperatorNode<>(left, right, operator); + } + throw new AssertionError(exprType); + } + + private Node generateBooleanExpression(int depth) { + List booleanExpressions = Arrays.stream(BooleanExpression.values()) + .collect(Collectors.toList()); + if (!globalState.getDbmsSpecificOptions().testBetween) { + booleanExpressions.remove(BooleanExpression.BETWEEN); + } + + booleanExpressions.remove(BooleanExpression.REGEX); + + BooleanExpression exprType = Randomly.fromList(booleanExpressions); + switch (exprType) { + case NOT: + return generateNOT(depth + 1); + case BINARY_COMPARISON: + return getBinaryComparison(depth); + case BINARY_LOGICAL: + return getBinaryLogical(depth); + case AND_OR_CHAIN: + return getAndOrChain(depth); + case REGEX: + return getRegex(depth); + case IS_NULL: + return new PrestoUnaryPostfixOperation(generateExpression(getRandomType(), depth + 1), + Randomly.fromOptions(PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL, + PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NOT_NULL)); + case IN: + return getInOperation(depth); + case BETWEEN: + return getBetween(depth); + case LIKE: + return getLike(depth); + case MULTI_VALUED_COMPARISON: // TODO other operators + return getMultiValuedComparison(depth); + default: + throw new AssertionError(exprType); + } + } + + private Node getMultiValuedComparison(int depth) { + + PrestoSchema.PrestoCompositeDataType type; + do { + type = PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getOrderableTypes())); + } while (type.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY + && !type.getElementType().getPrimitiveDataType().isOrderable()); + + PrestoMultiValuedComparisonType comparisonType = PrestoMultiValuedComparisonType.getRandom(); + PrestoMultiValuedComparisonOperator comparisonOperator = PrestoMultiValuedComparisonOperator + .getRandomForType(type); + Node left = generateExpression(type, depth + 1); + // sub-query + PrestoSchema.PrestoCompositeDataType finalType = type; + List columnsOfType = columns.stream().filter(c -> c.getType() == finalType) + .collect(Collectors.toList()); + if (Randomly.getBooleanWithRatherLowProbability() && !columnsOfType.isEmpty()) { + PrestoSchema.PrestoColumn column = Randomly.fromList(columnsOfType); + PrestoSelect subquery = generateSubquery(List.of(column)); + return new PrestoQuantifiedComparison(left, subquery, comparisonType, comparisonOperator); + } + int nr = Randomly.smallNumber() + 2; + List> rightList = new ArrayList<>(); + for (int i = 0; i < nr; i++) { + rightList.add(generateConstant(type)); + } + return new PrestoMultiValuedComparison(left, rightList, comparisonType, comparisonOperator); + } + + private PrestoSelect generateSubquery(List columns) { + PrestoSelect select = new PrestoSelect(); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + List tables = columns.stream().map(AbstractTableColumn::getTable) + .collect(Collectors.toList()); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).distinct() + .collect(Collectors.toList()); + List> tableNodeList = tables.stream() + .map(t -> new TableReferenceNode(t)) + .collect(Collectors.toList()); + select.setFromList(tableNodeList); + TypedExpressionGenerator, PrestoSchema.PrestoColumn, PrestoSchema.PrestoCompositeDataType> typedExpressionGenerator = new PrestoTypedExpressionGenerator( + globalState).setColumns(columns); + Node predicate = typedExpressionGenerator.generatePredicate(); + select.setWhereClause(predicate); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions(typedExpressionGenerator.generateOrderBys()); + } + List> joins = PrestoJoin.getJoins(tableList, globalState); + select.setJoinList(joins); + return select; + } + + private Node generateNumericExpression(int depth) { + PrestoSchema.PrestoDataType dataType = Randomly.fromList(PrestoSchema.PrestoDataType.getNumberTypes()); + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); + if (Randomly.getBoolean()) { + BinaryOperatorNode.Operator operator = PrestoBinaryArithmeticOperator.getRandom(); + Node left = generateExpression(type, depth); + Node right = generateExpression(type, depth); + return new NewBinaryOperatorNode<>(left, right, operator); + } else { + BinaryOperatorNode.Operator operator = PrestoUnaryArithmeticOperator.MINUS; + Node left = generateExpression(type, depth); + return new NewUnaryPrefixOperatorNode<>(left, operator); + } + } + + private Node generateTemporalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (Randomly.getBooleanWithSmallProbability()) { + Node left = generateExpression(type, depth); + Node right = generateExpression(PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getIntervalTypes())), depth); + BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); + return new NewBinaryOperatorNode<>(left, right, operator); + } + + // timestamp at time zone + if (Randomly.getBooleanWithSmallProbability() + && (type.getPrimitiveDataType() == PrestoSchema.PrestoDataType.TIMESTAMP + || type.getPrimitiveDataType() == PrestoSchema.PrestoDataType.TIMESTAMP_WITH_TIME_ZONE)) { + return new PrestoAtTimeZoneOperator(generateExpression(type, depth + 1), + PrestoConstant.createTimezoneConstant()); + } + return generateLeafNode(type); + } + + private Node generateIntervalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + if (Randomly.getBooleanWithSmallProbability()) { + Node left = generateExpression(type, depth); + + Node right; + if (Randomly.getBoolean()) { + right = generateExpression(PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getTemporalTypes())), depth); + } else { + right = generateExpression(type, depth); + } + BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); + if (Randomly.getBoolean()) { + return new NewBinaryOperatorNode<>(left, right, operator); + } else { + return new NewBinaryOperatorNode<>(right, left, operator); + } + } + return generateLeafNode(type); + + // functions + + // timestamp at time zone + } + + private Node getLike(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(PrestoSchema.PrestoDataType.VARCHAR); + Node expression = generateExpression(type, depth + 1); + Node pattern = generateExpression(type, depth + 1); + if (Randomly.getBoolean()) { + return new NewBinaryOperatorNode<>(expression, pattern, PrestoLikeOperator.getRandom()); + } else { + String randomlyString = randomly.getString(); + String randomlyChar = randomly.getChar(); + Node escape = new PrestoConstant.PrestoTextConstant(randomlyChar, 1); + int index = randomlyString.indexOf(randomlyChar); + while (index > -1) { + String wildcard = Randomly.fromOptions("%", "_"); + randomlyString = randomlyString.substring(0, index + 1) + wildcard + + randomlyString.substring(index + 1); + index = randomlyString.indexOf(randomlyChar, index + 1); + } + PrestoConstant.PrestoTextConstant patternString = new PrestoConstant.PrestoTextConstant(randomlyString); + return new NewTernaryNode<>(expression, patternString, escape, "LIKE", "ESCAPE"); + } + } + + private NewBinaryOperatorNode getRegex(int depth) { + Node left = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + Node right = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); + return new NewBinaryOperatorNode<>(left, right, PrestoBinaryLogicalOperator.getRandom()); + } + + private NewBinaryOperatorNode getBinaryLogical(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(PrestoSchema.PrestoDataType.BOOLEAN); + Node left = generateExpression(type, depth + 1); + Node right = generateExpression(type, depth + 1); + BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); + return new NewBinaryOperatorNode<>(left, right, operator); + } + + private Node getBetween(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getNumericTypes())); + Node expression = generateExpression(type, depth + 1); + Node left = generateExpression(type, depth + 1); + Node right = generateExpression(type, depth + 1); + return new NewBetweenOperatorNode<>(expression, left, right, Randomly.getBoolean()); + } + + private Node getInOperation(int depth) { + PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType + .fromDataType(PrestoSchema.PrestoDataType.getRandomWithoutNull()); + Node left = generateExpression(type, depth + 1); + List> inList = generateExpressions(type, Randomly.smallNumber() + 1, depth + 1); + boolean isNegated = Randomly.getBoolean(); + return new NewInOperatorNode<>(left, inList, isNegated); + } + + private Node getAndOrChain(int depth) { + Node left = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + Node right = generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); + BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); + left = new NewBinaryOperatorNode<>(left, right, operator); + } + return left; + } + + private Node getBinaryComparison(int depth) { + PrestoSchema.PrestoCompositeDataType type = getRandomType(); + BinaryOperatorNode.Operator op = PrestoBinaryComparisonOperator.getRandomForType(type); + Node left = generateExpression(type, depth + 1); + Node right = generateExpression(type, depth + 1); + return new NewBinaryOperatorNode<>(left, right, op); + } + + private Node generateNOT(int depth) { + PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator operator = PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT; + return new PrestoUnaryPrefixOperation(operator, generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth)); + } + + @Override + protected Node generateColumn(PrestoSchema.PrestoCompositeDataType type) { + List columnList = columns.stream() + .filter(c -> c.getType().getPrimitiveDataType() == type.getPrimitiveDataType()) + .collect(Collectors.toList()); + PrestoSchema.PrestoColumn column = Randomly.fromList(columnList); + return new PrestoColumnReference(column); + } + + @Override + public Node generateLeafNode(PrestoSchema.PrestoCompositeDataType type) { + if (Randomly.getBoolean()) { + return generateConstant(type); + } else { + List columnList = filterColumns(type.getPrimitiveDataType()); + if (columnList.isEmpty()) { + return generateConstant(type); + } else { + return generateColumn(type); + } + } + } + + private List filterColumns(PrestoSchema.PrestoDataType dataType) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType().getPrimitiveDataType() == dataType) + .collect(Collectors.toList()); + } + } + + @Override + protected PrestoSchema.PrestoCompositeDataType getRandomType() { + return PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull(); + } + + @Override + protected boolean canGenerateColumnOfType(PrestoSchema.PrestoCompositeDataType type) { + return columns.stream().anyMatch(c -> c.getType() == type); + } + + public Node generateAggregate() { + PrestoAggregateFunction aggregateFunction = PrestoAggregateFunction.getRandom(); + List> argsForAggregate = generateArgsForAggregate(aggregateFunction); + return new NewFunctionNode<>(argsForAggregate, aggregateFunction); + } + + public List> generateArgsForAggregate(PrestoAggregateFunction aggregateFunction) { + PrestoSchema.PrestoCompositeDataType returnType; + do { + returnType = aggregateFunction.getCompositeReturnType(); + } while (!aggregateFunction.isCompatibleWithReturnType(returnType)); + return aggregateFunction.getArgumentsForReturnType(this, this.maxDepth - 1, returnType, false); + } + + private Node generateAggregate(PrestoSchema.PrestoCompositeDataType type) { + PrestoAggregateFunction aggregateFunction = Randomly + .fromList(PrestoAggregateFunction.getFunctionsCompatibleWith(type)); + List> argsForAggregate = generateArgsForAggregate(type, aggregateFunction); + return new NewFunctionNode<>(argsForAggregate, aggregateFunction); + } + + public List> generateArgsForAggregate(PrestoSchema.PrestoCompositeDataType type, + PrestoAggregateFunction aggregateFunction) { + List returnTypes = aggregateFunction.getReturnTypes(type.getPrimitiveDataType()); + List> arguments = new ArrayList<>(); + allowAggregates = false; // + for (PrestoSchema.PrestoDataType argumentType : returnTypes) { + arguments.add(generateExpression(PrestoSchema.PrestoCompositeDataType.fromDataType(argumentType))); + } + // return new NewFunctionNode<>(arguments, aggregateFunction); + return arguments; + } + + @Override + public List> generateOrderBys() { + List> expressions = new ArrayList<>(); + int nr = Randomly.smallNumber() + 1; + ArrayList prestoColumns = new ArrayList<>(columns); + prestoColumns.removeIf(c -> !c.isOrderable()); + for (int i = 0; i < nr && !prestoColumns.isEmpty(); i++) { + PrestoSchema.PrestoColumn randomColumn = Randomly.fromList(prestoColumns); + PrestoColumnReference columnReference = new PrestoColumnReference(randomColumn); + prestoColumns.remove(randomColumn); + expressions.add(columnReference); + } + return expressions; + } + + public Node generateHavingClause() { + allowAggregates = true; + Node expr = generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + allowAggregates = false; + return expr; + } + + public Node generateExpressionWithColumns(List columns, + int remainingDepth) { + if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { + return generateConstant(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + } + PrestoSchema.PrestoColumn column = Randomly.fromList(columns); + if (remainingDepth <= 2 || Randomly.getBooleanWithRatherLowProbability()) { + return new PrestoColumnReference(column); + } + List possibleOptions = new ArrayList<>( + Arrays.asList(PrestoTypedExpressionGenerator.Expression.values())); + PrestoTypedExpressionGenerator.Expression expr = Randomly.fromList(possibleOptions); + BinaryOperatorNode.Operator op; + switch (expr) { + case BINARY_LOGICAL: + case BINARY_ARITHMETIC: + op = PrestoTypedExpressionGenerator.PrestoBinaryLogicalOperator.getRandom(); + break; + case BINARY_COMPARISON: + op = PrestoBinaryComparisonOperator.getRandom(); + break; + default: + throw new AssertionError(); + } + return new NewBinaryOperatorNode<>(generateExpression(column.getType(), remainingDepth - 1), + generateExpression(column.getType(), remainingDepth - 1), op); + } + + private enum StringExpression { + CONCAT + } + + public enum PrestBinaryStringOperator implements BinaryOperatorNode.Operator { + CONCAT("||"); + + private final String textRepresentation; + + PrestBinaryStringOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoBinaryTemporalOperator implements BinaryOperatorNode.Operator { + ADD("+"), SUB("-"); + + private final String textRepresentation; + + PrestoBinaryTemporalOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + private enum BooleanExpression { + NOT, BINARY_COMPARISON, BINARY_LOGICAL, AND_OR_CHAIN, REGEX, IS_NULL, IN, BETWEEN, LIKE, MULTI_VALUED_COMPARISON + } + + public enum PrestoBinaryLogicalOperator implements BinaryOperatorNode.Operator { + + AND, OR; + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return toString(); + } + + } + + public enum PrestoLikeOperator implements BinaryOperatorNode.Operator { + LIKE("LIKE"), // + NOT_LIKE("NOT LIKE"); + + private final String textRepresentation; + + PrestoLikeOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static PrestoLikeOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoBinaryComparisonOperator implements BinaryOperatorNode.Operator { + EQUALS("="), NOT_EQUALS("<>"), NOT_EQUALS_ALT("!="), IS_DISTINCT_FROM("IS DISTINCT FROM"), + IS_NOT_DISTINCT_FROM("IS NOT DISTINCT FROM"), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), + SMALLER_EQUALS("<="); + + private final String textRepresentation; + + PrestoBinaryComparisonOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + public static BinaryOperatorNode.Operator getRandomStringOperator() { + return Randomly.fromOptions(EQUALS, NOT_EQUALS, IS_DISTINCT_FROM, IS_NOT_DISTINCT_FROM); + } + + public static BinaryOperatorNode.Operator getRandomForType(PrestoSchema.PrestoCompositeDataType type) { + PrestoSchema.PrestoDataType dataType = type.getPrimitiveDataType(); + + switch (dataType) { + case BOOLEAN: + case INT: + case FLOAT: + case DECIMAL: + case DATE: + case TIME: + case TIMESTAMP: + case TIME_WITH_TIME_ZONE: + case TIMESTAMP_WITH_TIME_ZONE: + return getRandom(); + case VARCHAR: + case CHAR: + case VARBINARY: + case JSON: + case ARRAY: + case INTERVAL_YEAR_TO_MONTH: + case INTERVAL_DAY_TO_SECOND: + // return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, IS_DISTINCT_FROM, + // IS_NOT_DISTINCT_FROM); + default: + return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, IS_DISTINCT_FROM, IS_NOT_DISTINCT_FROM); + } + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoBinaryArithmeticOperator implements BinaryOperatorNode.Operator { + ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"); + + private final String textRepresentation; + + PrestoBinaryArithmeticOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + public static BinaryOperatorNode.Operator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + public enum PrestoUnaryArithmeticOperator implements BinaryOperatorNode.Operator { + MINUS("-"); + + private final String textRepresentation; + + PrestoUnaryArithmeticOperator(String textRepresentation) { + this.textRepresentation = textRepresentation; + } + + @Override + public String getTextRepresentation() { + return textRepresentation; + } + + } + + private enum Expression { + BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC + } + +} diff --git a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java new file mode 100644 index 000000000..bdd8cf2a3 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java @@ -0,0 +1,53 @@ +package sqlancer.presto.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.AbstractUpdateGenerator; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public final class PrestoUpdateGenerator extends AbstractUpdateGenerator { + + private final PrestoGlobalState globalState; + private PrestoTypedExpressionGenerator gen; + + private PrestoUpdateGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + return new PrestoUpdateGenerator(globalState).generate(); + } + + private SQLQueryAdapter generate() { + PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + List columns = table.getRandomNonEmptyColumnSubset(); + gen = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()); + sb.append("UPDATE "); + sb.append(table.getName()); + sb.append(" SET "); + updateColumns(columns); + PrestoErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } + + @Override + protected void updateValue(PrestoColumn column) { + Node expr; + if (Randomly.getBooleanWithSmallProbability()) { + expr = gen.generateExpression(column.getType()); + PrestoErrors.addExpressionErrors(errors); + } else { + expr = gen.generateConstant(column.getType()); + } + sb.append(PrestoToStringVisitor.asString(expr)); + } + +} diff --git a/src/sqlancer/presto/gen/PrestoViewGenerator.java b/src/sqlancer/presto/gen/PrestoViewGenerator.java new file mode 100644 index 000000000..72130cf82 --- /dev/null +++ b/src/sqlancer/presto/gen/PrestoViewGenerator.java @@ -0,0 +1,36 @@ +package sqlancer.presto.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoToStringVisitor; + +public final class PrestoViewGenerator { + + private PrestoViewGenerator() { + } + + public static SQLQueryAdapter generate(PrestoGlobalState globalState) { + int nrColumns = Randomly.smallNumber() + 1; + StringBuilder sb = new StringBuilder("CREATE "); + sb.append("VIEW "); + sb.append(globalState.getSchema().getFreeViewName()); + sb.append("("); + for (int i = 0; i < nrColumns; i++) { + if (i != 0) { + sb.append(", "); + } + sb.append("c"); + sb.append(i); + } + sb.append(") AS "); + sb.append(PrestoToStringVisitor.asString(PrestoRandomQuerySynthesizer.generateSelect(globalState, nrColumns))); + ExpectedErrors errors = new ExpectedErrors(); + PrestoErrors.addExpressionErrors(errors); + PrestoErrors.addGroupByErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + +} diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java new file mode 100644 index 000000000..a63d912b0 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -0,0 +1,138 @@ +package sqlancer.presto.test; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewPostfixTextNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public class PrestoNoRECOracle extends NoRECBase implements TestOracle { + + private final PrestoSchema s; + + public PrestoNoRECOracle(PrestoGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + PrestoErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + PrestoTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + + List tables = randomTables.getTables(); + + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = PrestoJoin.getJoins(tableList, state); + + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(state).setColumns(columns); + Node randomWhereCondition = gen.generatePredicate(); + int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); + + int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); + if (firstCount == -1 || secondCount == -1) { + throw new IgnoreMeException(); + } + if (firstCount != secondCount) { + throw new AssertionError( + optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); + } + } + + private int getSecondQuery(List> tableList, Node randomWhereCondition, + List> joins) throws SQLException { + PrestoSelect select = new PrestoSelect(); + + Node asText = new NewPostfixTextNode<>( + + new PrestoCastFunction( + new NewPostfixTextNode<>(randomWhereCondition, + " IS NOT NULL AND " + PrestoToStringVisitor.asString(randomWhereCondition)), + new PrestoCompositeDataType(PrestoDataType.INT, 8, 0)), + "as count"); + + select.setFetchColumns(List.of(asText)); + select.setFromList(tableList); + select.setJoinList(joins); + int secondCount = 0; + unoptimizedQueryString = "SELECT SUM(count) FROM (" + PrestoToStringVisitor.asString(select) + ") as res"; + + errors.add("canceling statement due to statement timeout"); + SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors, false, false); + SQLancerResultSet rs; + try { + rs = q.executeAndGetLogged(state); + } catch (Exception e) { + throw new AssertionError(unoptimizedQueryString, e); + } + if (rs == null) { + return -1; + } + if (rs.next()) { + secondCount += rs.getLong(1); + } + rs.close(); + return secondCount; + } + + private int getFirstQueryCount(SQLConnection con, List> tableList, + List columns, Node randomWhereCondition, + List> joins) { + PrestoSelect select = new PrestoSelect(); + List> allColumns = columns.stream() + .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + select.setFetchColumns(allColumns); + select.setFromList(tableList); + select.setWhereClause(randomWhereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByExpressions( + new PrestoTypedExpressionGenerator(state).setColumns(columns).generateOrderBys()); + } + select.setJoinList(joins); + int firstCount = 0; + try (Statement stat = con.createStatement()) { + optimizedQueryString = PrestoToStringVisitor.asString(select); + if (options.logEachSelect()) { + logger.writeCurrent(optimizedQueryString); + } + try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { + while (rs.next()) { + firstCount++; + } + } + } catch (SQLException e) { + throw new IgnoreMeException(); + } + return firstCount; + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java new file mode 100644 index 000000000..527d5d648 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java @@ -0,0 +1,206 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewAliasNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoAggregateFunction; +import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoUnaryPostfixOperation; +import sqlancer.presto.ast.PrestoUnaryPrefixOperation; + +public class PrestoQueryPartitioningAggregateTester extends PrestoQueryPartitioningBase + implements TestOracle { + + private String firstResult; + private String firstResultType; + private String secondResult; + private String originalQuery; + private String metamorphicQuery; + + public PrestoQueryPartitioningAggregateTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + PrestoErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + PrestoAggregateFunction aggregateFunction = Randomly.fromOptions(PrestoAggregateFunction.MAX, + PrestoAggregateFunction.MIN, PrestoAggregateFunction.SUM, PrestoAggregateFunction.COUNT, + PrestoAggregateFunction.AVG/* , PrestoAggregateFunction.STDDEV_POP */); + List> aggregateArgs = gen.generateArgsForAggregate(aggregateFunction); + NewFunctionNode aggregate = new NewFunctionNode<>(aggregateArgs, + aggregateFunction); + select.setFetchColumns(List.of(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + originalQuery = PrestoToStringVisitor.asString(select); + firstResult = getAggregateResult(originalQuery); + firstResultType = getAggregateResultType(originalQuery); + metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); + secondResult = getAggregateResult(metamorphicQuery); + + state.getState().getLocalState().log( + "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); + if (firstResultType.equals("VARBINARY") || firstResultType.equals("ARRAY(VARBINARY)") + || firstResultType.equals("ARRAY(ARRAY(VARBINARY))")) { + throw new IgnoreMeException(); + } + if (firstResult == null && secondResult != null) { + if (secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + throw new AssertionError(); + } else if (firstResult != null && !firstResult.contentEquals(secondResult) + && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { + if (secondResult.contains("Inf")) { + throw new IgnoreMeException(); // FIXME: average computation + } + throw new AssertionError(); + } + + } + + private String createMetamorphicUnionQuery(PrestoSelect select, + NewFunctionNode aggregate, List> from) { + String metamorphicQuery; + Node whereClause = gen.generatePredicate(); + Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT); + Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); + List> mappedAggregate = mapped(aggregate); + PrestoSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); + PrestoSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); + PrestoSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; + metamorphicQuery += PrestoToStringVisitor.asString(leftSelect) + " UNION ALL " + + PrestoToStringVisitor.asString(middleSelect) + " UNION ALL " + + PrestoToStringVisitor.asString(rightSelect); + metamorphicQuery += ") as asdf"; + return metamorphicQuery; + } + + private String getAggregateResult(String queryString) { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getString(1); + } + return resultString; + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } + + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private String getAggregateResultType(String queryString) { + String resultString; + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); + try (SQLancerResultSet result = q.executeAndGet(state)) { + if (result == null) { + throw new IgnoreMeException(); + } + if (!result.next()) { + resultString = null; + } else { + resultString = result.getType(1); + } + return resultString; + } catch (SQLException e) { + if (!e.getMessage().contains("Not implemented type")) { + throw new AssertionError(queryString, e); + } else { + throw new IgnoreMeException(); + } + } + } + + private List> mapped(NewFunctionNode aggregate) { + PrestoCastFunction count; + switch (aggregate.getFunc()) { + case COUNT: + case MAX: + case MIN: + case SUM: + return aliasArgs(List.of(aggregate)); + case AVG: + NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + PrestoAggregateFunction.SUM); + count = new PrestoCastFunction(new NewFunctionNode<>(aggregate.getArgs(), PrestoAggregateFunction.COUNT), + new PrestoCompositeDataType(PrestoDataType.FLOAT, 8, 0)); + return aliasArgs(Arrays.asList(sum, count)); + default: + throw new AssertionError(aggregate.getFunc()); + } + } + + private List> aliasArgs(List> originalAggregateArgs) { + List> args = new ArrayList<>(); + int i = 0; + for (Node expr : originalAggregateArgs) { + args.add(new NewAliasNode<>(expr, "agg" + i++)); + } + return args; + } + + private String getOuterAggregateFunction(NewFunctionNode aggregate) { + switch (aggregate.getFunc()) { + case AVG: + return "SUM(CAST(agg0 AS DOUBLE))/CAST(SUM(agg1) AS DOUBLE)"; + case COUNT: + return PrestoAggregateFunction.SUM + "(agg0)"; + default: + return aggregate.getFunc().toString() + "(agg0)"; + } + } + + private PrestoSelect getSelect(List> aggregates, List> from, + Node whereClause, List> joinList) { + PrestoSelect leftSelect = new PrestoSelect(); + leftSelect.setFetchColumns(aggregates); + leftSelect.setFromList(from); + leftSelect.setWhereClause(whereClause); + leftSelect.setJoinList(joinList); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + } + return leftSelect; + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java new file mode 100644 index 000000000..5ee56e1c8 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java @@ -0,0 +1,90 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; + +public class PrestoQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, PrestoGlobalState> + implements TestOracle { + + PrestoSchema s; + PrestoTables targetTables; + PrestoTypedExpressionGenerator gen; + PrestoSelect select; + + public PrestoQueryPartitioningBase(PrestoGlobalState state) { + super(state); + PrestoErrors.addExpressionErrors(errors); + } + + public static String canonicalizeResultValue(String value) { + if (value == null) { + return null; + } + + // TODO: check this + switch (value) { + case "-0.0": + return "0.0"; + case "-0": + return "0"; + default: + } + + return value; + } + + @Override + public void check() throws SQLException { + s = state.getSchema(); + targetTables = s.getRandomTableNonEmptyTables(); + gen = new PrestoTypedExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new PrestoSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List> joins = PrestoJoin.getJoins(tableList, state); + select.setJoinList(new ArrayList<>(joins)); + select.setFromList(new ArrayList<>(tableList)); + select.setWhereClause(null); + } + + List> generateFetchColumns() { + List> columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new ColumnReferenceNode<>(new PrestoColumn("*", null, false, false))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + return columns; + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java new file mode 100644 index 000000000..008e8f4db --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningDistinctTester.java @@ -0,0 +1,44 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoToStringVisitor; + +public class PrestoQueryPartitioningDistinctTester extends PrestoQueryPartitioningBase { + + public PrestoQueryPartitioningDistinctTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setDistinct(true); + select.setWhereClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + if (Randomly.getBoolean()) { + select.setDistinct(false); + } + select.setWhereClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java new file mode 100644 index 000000000..5d5e2343f --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java @@ -0,0 +1,53 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoQueryPartitioningGroupByTester extends PrestoQueryPartitioningBase { + + public PrestoQueryPartitioningGroupByTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setGroupByExpressions(select.getFetchColumns()); + select.setWhereClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setWhereClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, + secondQueryString, thirdQueryString, combinedString, true, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + List> generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream() + .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java new file mode 100644 index 000000000..b3ff3beaa --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java @@ -0,0 +1,64 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.oracle.TestOracle; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoExpression; + +public class PrestoQueryPartitioningHavingTester extends PrestoQueryPartitioningBase + implements TestOracle { + + public PrestoQueryPartitioningHavingTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull())); + } + boolean orderBy = Randomly.getBoolean(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setHavingClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setHavingClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setHavingClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setHavingClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + + @Override + protected Node generatePredicate() { + return gen.generateHavingClause(); + } + + @Override + List> generateFetchColumns() { + return Collections.singletonList(gen.generateHavingClause()); + } + +} diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java new file mode 100644 index 000000000..093c27108 --- /dev/null +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java @@ -0,0 +1,46 @@ +package sqlancer.presto.test; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.presto.PrestoErrors; +import sqlancer.presto.PrestoGlobalState; +import sqlancer.presto.PrestoToStringVisitor; + +public class PrestoQueryPartitioningWhereTester extends PrestoQueryPartitioningBase { + + public PrestoQueryPartitioningWhereTester(PrestoGlobalState state) { + super(state); + PrestoErrors.addGroupByErrors(errors); + PrestoErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + super.check(); + select.setWhereClause(null); + String originalQueryString = PrestoToStringVisitor.asString(select); + + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + select.setWhereClause(predicate); + String firstQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PrestoToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PrestoToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, PrestoQueryPartitioningBase::canonicalizeResultValue); + } + +} From 70d11138eb01a5253ada440ee9a469bb8784e871 Mon Sep 17 00:00:00 2001 From: ZHENGLIN LI <63448884+ZhengLin-Li@users.noreply.github.com> Date: Tue, 5 Sep 2023 10:26:23 +0800 Subject: [PATCH 391/856] feat: add support of 8.0 version (#886) * feat: add support of 8.0 version * feat: add expected errors for 8.0 version * fix: change --test-8.0-version to true * feat: add RENAME COLUMN support when altering table * fix: test error * feat: add AS support when inserting * feat: add AS support when deleting --- src/sqlancer/stonedb/StoneDBErrors.java | 9 ++++++++- src/sqlancer/stonedb/StoneDBOptions.java | 3 +++ src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java | 6 ++++++ .../stonedb/gen/StoneDBTableDeleteGenerator.java | 4 ++++ .../stonedb/gen/StoneDBTableInsertGenerator.java | 9 +++++++++ src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java | 2 +- 8 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java index 1e72e5685..152c5bfc9 100644 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -1,12 +1,13 @@ package sqlancer.stonedb; import sqlancer.common.query.ExpectedErrors; +import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; public final class StoneDBErrors { private StoneDBErrors() { } - public static void addExpectedExpressionErrors(ExpectedErrors errors) { + public static void addExpectedExpressionErrors(StoneDBGlobalState globalState, ExpectedErrors errors) { // java.sql.SQLException: Incorrect DATE value: '292269055-12-02' errors.add("Incorrect DATE value: "); // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 @@ -31,5 +32,11 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("Numeric result of an expression is too large and cannot be handled by tianmu."); // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order clause' errors.add("Unknown column "); + if (globalState.getDbmsSpecificOptions().test80Version) { + // Caused by: java.sql.SQLException: Incorrect DATETIME value: '292269055-12-02 16:47:04' + errors.add("Incorrect DATETIME value: "); + // Caused by: java.sql.SQLException: Incorrect TIMESTAMP value: '292269055-12-02 16:47:04' + errors.add("Incorrect TIMESTAMP value: "); + } } } diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java index a9728323f..0c9d8816a 100644 --- a/src/sqlancer/stonedb/StoneDBOptions.java +++ b/src/sqlancer/stonedb/StoneDBOptions.java @@ -27,6 +27,9 @@ public class StoneDBOptions implements DBMSSpecificOptions public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 3306; + @Parameter(names = "--test-8-version", description = "Let SQLancer test the StoneDB 8.0, otherwise, SQLancer will test the StoneDB 5.7", arity = 1) + public boolean test80Version = true; + @Parameter(names = "--oracle") public List oracles = List.of(StoneDBOracleFactory.NOREC); diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java index e9c283cc9..c37024669 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java @@ -66,6 +66,12 @@ private void addExpectedErrors() { private void appendAlterOption(Action action) { StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) .setColumns(table.getColumns()); + if (globalState.getDbmsSpecificOptions().test80Version && Randomly.getBooleanWithSmallProbability()) { + sb.append("RENAME COLUMN "); + sb.append(table.getRandomColumn().getName()); + sb.append(" TO "); + sb.append(table.getFreeColumnName()); + } switch (action) { case ADD_COLUMN: sb.append("ADD COLUMN "); diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java index 16b9c1061..0d8446d4b 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java @@ -40,6 +40,10 @@ public SQLQueryAdapter getQuery() { } sb.append(" FROM "); sb.append(randomTable.getName()); + if (globalState.getDbmsSpecificOptions().test80Version) { + sb.append(" AS "); + sb.append(globalState.getSchema().getFreeTableName()); + } if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java index a9823be6d..0a70c8467 100644 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java @@ -48,6 +48,7 @@ private SQLQueryAdapter getQuery() { sb.append(table.getName()); appendPartition(); appendColumnsAndValues(columns); + appendAS(); appendOnDuplicateUpdate(); addExpectedErrors(); return new SQLQueryAdapter(sb.toString(), errors); @@ -84,6 +85,14 @@ private void appendColumnsAndValues(List columns) { appendValues(); } + private void appendAS() { + if (!globalState.getDbmsSpecificOptions().test80Version) { + return; + } + sb.append(" AS "); + sb.append("r").append(table.getNrRows(globalState)); + } + private void appendOnDuplicateUpdate() { sb.append("on duplicate key update "); StoneDBColumn randomColumn = table.getRandomColumn(); diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java index 27ba91013..2465494ef 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -31,7 +31,7 @@ public class StoneDBAggregateOracle extends StoneDBQueryPartitioningBase { public StoneDBAggregateOracle(StoneDBGlobalState state) { super(state); - StoneDBErrors.addExpectedExpressionErrors(errors); + StoneDBErrors.addExpectedExpressionErrors(state, errors); } @Override diff --git a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java index 7e4d6a622..6cd5b3bc9 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java @@ -31,7 +31,7 @@ public class StoneDBFuzzOracle implements TestOracle { public StoneDBFuzzOracle(StoneDBGlobalState globalState) { this.globalState = globalState; this.schema = globalState.getSchema(); - StoneDBErrors.addExpectedExpressionErrors(errors); + StoneDBErrors.addExpectedExpressionErrors(globalState, errors); } @Override diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index e042cd44a..744d52f55 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -43,7 +43,7 @@ public class StoneDBNoRECOracle extends NoRECBase implements public StoneDBNoRECOracle(StoneDBGlobalState globalState) { super(globalState); this.schema = globalState.getSchema(); - StoneDBErrors.addExpectedExpressionErrors(errors); + StoneDBErrors.addExpectedExpressionErrors(globalState, errors); } @Override From 952c7cd37d065950c9c0fb76721c3aac7684983f Mon Sep 17 00:00:00 2001 From: Nathee Jaywaree <73115539+eltfshr@users.noreply.github.com> Date: Tue, 5 Sep 2023 13:56:12 +0700 Subject: [PATCH 392/856] Add Fuzzer oracle for PostgreSQL and MySQL (#893) * Add random query synthesizer for MySQL * Add Fuzzer oracle for MySQL and Postgres --- src/sqlancer/mysql/MySQLOptions.java | 8 +++ .../gen/MySQLRandomQuerySynthesizer.java | 52 +++++++++++++++++++ src/sqlancer/mysql/oracle/MySQLFuzzer.java | 30 +++++++++++ src/sqlancer/postgres/PostgresOptions.java | 8 +++ .../postgres/oracle/PostgresFuzzer.java | 30 +++++++++++ src/sqlancer/sqlite3/SQLite3Options.java | 4 -- .../sqlite3/oracle/SQLite3Fuzzer.java | 6 +-- 7 files changed, 130 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java create mode 100644 src/sqlancer/mysql/oracle/MySQLFuzzer.java create mode 100644 src/sqlancer/postgres/oracle/PostgresFuzzer.java diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 2d9e80bc0..4e42e8b00 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -12,6 +12,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; import sqlancer.mysql.oracle.MySQLCERTOracle; +import sqlancer.mysql.oracle.MySQLFuzzer; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @@ -57,6 +58,13 @@ public TestOracle create(MySQLGlobalState globalState) throws public boolean requiresAllTablesToContainRows() { return true; } + }, + FUZZER { + @Override + public TestOracle create(MySQLGlobalState globalState) throws Exception { + return new MySQLFuzzer(globalState); + } + }; } diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java new file mode 100644 index 000000000..686e6091d --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -0,0 +1,52 @@ +package sqlancer.mysql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLTableReference; + +public final class MySQLRandomQuerySynthesizer { + + private MySQLRandomQuerySynthesizer() { + } + + public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) { + MySQLTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + MySQLSelect select = new MySQLSelect(); + List columns = new ArrayList<>(); + + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); + columns.addAll(gen.generateExpressions(nrColumns)); + select.setFetchColumns(columns); + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) + .collect(Collectors.toList()); + select.setFromList(tableList); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateHavingClause()); + } + } + if (Randomly.getBoolean()) { + select.setLimitClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(gen.generateExpression()); + } + return select; + } + +} diff --git a/src/sqlancer/mysql/oracle/MySQLFuzzer.java b/src/sqlancer/mysql/oracle/MySQLFuzzer.java new file mode 100644 index 000000000..2e361c0b8 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLFuzzer.java @@ -0,0 +1,30 @@ +package sqlancer.mysql.oracle; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.gen.MySQLRandomQuerySynthesizer; + +public class MySQLFuzzer implements TestOracle { + + private final MySQLGlobalState globalState; + + public MySQLFuzzer(MySQLGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public void check() throws Exception { + String s = MySQLVisitor.asString(MySQLRandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + + ';'; + try { + globalState.executeStatement(new SQLQueryAdapter(s)); + globalState.getManager().incrementSelectQueryCount(); + } catch (Error e) { + + } + } + +} diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 948a539ce..bb76404de 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -13,6 +13,7 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; +import sqlancer.postgres.oracle.PostgresFuzzer; import sqlancer.postgres.oracle.PostgresNoRECOracle; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; @@ -76,6 +77,13 @@ public TestOracle create(PostgresGlobalState globalState) t oracles.add(new PostgresTLPAggregateOracle(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + FUZZER { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + return new PostgresFuzzer(globalState); + } + }; } diff --git a/src/sqlancer/postgres/oracle/PostgresFuzzer.java b/src/sqlancer/postgres/oracle/PostgresFuzzer.java new file mode 100644 index 000000000..a0b46bd17 --- /dev/null +++ b/src/sqlancer/postgres/oracle/PostgresFuzzer.java @@ -0,0 +1,30 @@ +package sqlancer.postgres.oracle; + +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.gen.PostgresRandomQueryGenerator; + +public class PostgresFuzzer implements TestOracle { + + private final PostgresGlobalState globalState; + + public PostgresFuzzer(PostgresGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public void check() throws Exception { + String s = PostgresVisitor.asString( + PostgresRandomQueryGenerator.createRandomQuery(Randomly.smallNumber() + 1, globalState)) + ';'; + try { + globalState.executeStatement(new SQLQueryAdapter(s)); + globalState.getManager().incrementSelectQueryCount(); + } catch (Error e) { + + } + } + +} diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index e740a5eea..e2c04ed46 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -86,10 +86,6 @@ public class SQLite3Options implements DBMSSpecificOptions "--generate-new-database" }, description = "Specifies whether new databases should be generated", arity = 1) public boolean generateDatabase = true; - @Parameter(names = { - "--execute-queries" }, description = "Specifies whether the query in the fuzzer should be executed", arity = 1) - public boolean executeQuery = true; - @Parameter(names = { "--max-num-tables" }, description = "The maximum number of tables/virtual tables/ rtree tables/ views that can be created") public int maxNumTables = 10; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java index bf13ffc80..e98e5f032 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3Fuzzer.java @@ -20,10 +20,8 @@ public void check() throws Exception { String s = SQLite3Visitor .asString(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)) + ";"; try { - if (globalState.getDbmsSpecificOptions().executeQuery) { - globalState.executeStatement(new SQLQueryAdapter(s)); - globalState.getManager().incrementSelectQueryCount(); - } + globalState.executeStatement(new SQLQueryAdapter(s)); + globalState.getManager().incrementSelectQueryCount(); } catch (Error e) { } From cddff69308e9bcdbf0ed9abc5b77b57590431ff7 Mon Sep 17 00:00:00 2001 From: Nathee Jaywaree <73115539+eltfshr@users.noreply.github.com> Date: Fri, 15 Sep 2023 08:39:53 +0700 Subject: [PATCH 393/856] Change OFFSET and LIMIT clause in MySQLRandomQuerySynthesizer (#894) * Add random query synthesizer for MySQL * Add Fuzzer oracle for MySQL and Postgres * Change offset from MySQLRandomQuerySynthesizer --- src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java index 686e6091d..8ebaff3b6 100644 --- a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -7,6 +7,7 @@ import sqlancer.Randomly; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.ast.MySQLConstant; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLTableReference; @@ -41,10 +42,10 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) } } if (Randomly.getBoolean()) { - select.setLimitClause(gen.generateExpression()); - } - if (Randomly.getBoolean()) { - select.setOffsetClause(gen.generateExpression()); + select.setLimitClause(MySQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(MySQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } } return select; } From b9759b4ae8ed9cdee06cbac7a4ba6477be3b24d2 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 6 Nov 2023 18:32:31 +0800 Subject: [PATCH 394/856] Reducer updates. (#892) * use single `FileWriter` for reducer. * Trivial update. * Add warning when reducer is not available. * resolve comment. * Try to implement a reducer. * fix format * Bump `setup-java` to v3 to cache maven artifacts. * fix PMD. * Remove non-LTS Java versions. * Fix reducer log. * Resolve comments. * Update doc. * Extract reproducer to separate class. * Fix format issues. * Cleanup and update messages. --- .github/workflows/main.yml | 178 +++++++++--------- .github/workflows/release.yml | 10 +- docs/testCaseReduction.md | 23 ++- src/sqlancer/ASTBasedReducer.java | 3 +- src/sqlancer/Main.java | 54 ++++-- src/sqlancer/ProviderAdapter.java | 5 +- .../sqlite3/oracle/SQLite3NoRECOracle.java | 22 +-- .../tidb/oracle/TiDBTLPWhereOracle.java | 44 +++++ 8 files changed, 204 insertions(+), 135 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ec8698f03..612c6f571 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,9 +24,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests @@ -46,9 +48,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up Citus @@ -92,9 +96,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up ClickHouse @@ -119,9 +125,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up CockroachDB @@ -142,9 +150,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up CockroachDB @@ -174,9 +184,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -192,9 +204,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: DuckDB Tests @@ -208,9 +222,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -225,9 +241,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Install MariaDB @@ -257,9 +275,11 @@ jobs: # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -282,9 +302,11 @@ jobs: # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -300,9 +322,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up MySQL @@ -330,9 +354,11 @@ jobs: postgresql password: 'sqlancer' postgresql db: 'test' - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests @@ -347,9 +373,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests @@ -366,9 +394,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: SQLite Tests for QPG @@ -393,9 +423,11 @@ jobs: docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" sleep 60s - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: StoneDB Tests @@ -410,9 +442,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up TiDB @@ -433,9 +467,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up TiDB @@ -456,9 +492,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Set up Yugabyte @@ -478,9 +516,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: install mysql client run: | sudo apt update @@ -509,58 +549,6 @@ jobs: - name: Run Tests run: DORIS_AVAILABLE=true mvn -Dtest=TestDoris test - - java13: - name: Java 13 Compatibility (DuckDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up JDK 13 - uses: actions/setup-java@v1.4.4 - with: - java-version: 13 - - name: Build - run: mvn -B package -DskipTests=true - - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - - java14: - name: Java 14 Compatibility (DuckDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up JDK 14 - uses: actions/setup-java@v1.4.4 - with: - java-version: 14 - - name: Build - run: mvn -B package -DskipTests=true - - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - - java15: - name: Java 15 EA Compatibility (DuckDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up JDK 15 - uses: actions/setup-java@v1.4.4 - with: - java-version: 15-ea - - name: Build - run: mvn -B package -DskipTests=true - - name: Shortly run DuckDB - run: cd target && java -jar $(ls | grep -P 'sqlancer-[0-9.]*.jar') --num-threads 4 --timeout-seconds 30 --num-queries 0 duckdb - reducer: name: Reducer Tests runs-on: ubuntu-latest @@ -570,9 +558,11 @@ jobs: with: fetch-depth: 0 - name: Set up JDK 11 - uses: actions/setup-java@v1.4.4 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' + cache: 'maven' - name: Build run: mvn -B package -DskipTests=true - name: Run Tests diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 342a358f3..6194c1529 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,9 +9,10 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Maven Central Repository - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD @@ -29,9 +30,10 @@ jobs: - name: Check out the repo uses: actions/checkout@v2 - name: Set up JDK 11 - uses: actions/setup-java@v1 + uses: actions/setup-java@v3 with: - java-version: 11 + distribution: 'temurin' + java-version: '11' - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Push to Docker Hub diff --git a/docs/testCaseReduction.md b/docs/testCaseReduction.md index c95770688..ee317f791 100644 --- a/docs/testCaseReduction.md +++ b/docs/testCaseReduction.md @@ -11,7 +11,26 @@ The AST-based reducer can shorten a statement by applying AST level transformati The transformations are implemented by [JSQLParser](https://github.com/JSQLParser/JSqlParser), a RDBMS agnostic SQL statement parser that can translate SQL statements into a traversable hierarchy of Java classes. JSQLParser provides support for the SQL standard as well as major SQL dialects. The AST-based reducer works for any SQL dialects that can be parsed by this tool. -## Enable reducers +## Implementing reproducer +Determining whether a bug persists after reducing statements +is an undecidable task for general transformations. +In practice, reducers use the [reproducer](../src/sqlancer/Reproducer.java) to determine +if a bug remains after statements have been removed or modified. +The reducer's responsibility is to verify if the current state, +formed by the pared-down statements, +continues to yield incorrect results for specific queries. + +Different oracles have distinct logic for determination, +meaning a universal reproducer doesn't exist. +Each oracle type needs its own reproducer implementation. +If reproducer is not implemented for specific oracle, +test case reduction is not available while using the oracle. + +Oracles for which reproducers have currently been implemented include: +1. for [`SQLite3NoRECOracle`](../src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java) +2. for [`TiDBTLPWhereOracle`](../src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java) + +## Using reducers Test-case reduction is disabled by default. The statement reducer can be enabled by passing `--use-reducer` when starting SQLancer. If you wish to further shorten each statements, you need to additionally pass the `--reduce-ast` parameter so that the AST-based reduction is applied. Note: if `--reduce-ast` is set, `--use-reducer` option must be enabled first. @@ -28,4 +47,4 @@ There are also options to define timeout seconds and max steps of reduction for ## Reduction logs If test-case reduction is enabled, each time the reducer performs a reduction step successfully,it prints the reduced statements to the log file, overwriting the previous ones. -The log files will be stored in the following format: `logs//reduce/-reduce.log`. For instance, if the tested DBMS is SQLite3 and the current database is named database0, the log file will be located at `logs/sqlite3/reduce/database0-reduce.log`. \ No newline at end of file +The log files will be stored in the following format: `logs//reduce/-reduce.log`. For instance, if the tested DBMS is SQLite3 and the current database is named database0, the log file will be located at `logs/sqlite3/reduce/database0-reduce.log`. diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java index 36f16c2ac..876a2da12 100644 --- a/src/sqlancer/ASTBasedReducer.java +++ b/src/sqlancer/ASTBasedReducer.java @@ -103,7 +103,8 @@ public void reduce(G state, Reproducer reproducer, G newGlobalState) throws E }); if (!initFlag) { - System.out.println("Error when parsing the statement at transformer :" + t); + newGlobalState.getLogger() + .logReducer("warning: failed parsing the statement at transformer : " + t); continue; } t.apply(); diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 89fbafd38..ebb3f1213 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -81,6 +81,7 @@ public static final class StateLogger { private FileWriter logFileWriter; public FileWriter currentFileWriter; private FileWriter queryPlanFileWriter; + private FileWriter reduceFileWriter; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; @@ -201,14 +202,14 @@ public FileWriter getReduceFileWriter() { if (!useReducer) { throw new UnsupportedOperationException(); } - FileWriter fileWriter; - try { - fileWriter = new FileWriter(reduceFile, false); - } catch (IOException e) { - throw new AssertionError(e); + if (reduceFileWriter == null) { + try { + reduceFileWriter = new FileWriter(reduceFile, false); + } catch (IOException e) { + throw new AssertionError(e); + } } - - return fileWriter; + return reduceFileWriter; } public void writeCurrent(StateToReproduce state) { @@ -257,6 +258,26 @@ public void writeQueryPlan(String queryPlan) { } } + public void logReducer(String reducerLog) { + FileWriter reduceFileWriter = getReduceFileWriter(); + + StringBuilder sb = new StringBuilder(); + sb.append("[reducer log] "); + sb.append(reducerLog); + try { + reduceFileWriter.write(sb.toString()); + } catch (IOException e) { + throw new AssertionError(e); + } finally { + try { + reduceFileWriter.flush(); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + } + public void logReduced(StateToReproduce state) { FileWriter reduceFileWriter = getReduceFileWriter(); @@ -272,7 +293,6 @@ public void logReduced(StateToReproduce state) { } finally { try { reduceFileWriter.flush(); - reduceFileWriter.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); @@ -293,7 +313,6 @@ public void logException(Throwable reduce, StateToReproduce state) { try { logFileWriter2.flush(); } catch (IOException e) { - // TODO Auto-generated catch block e.printStackTrace(); } } @@ -449,9 +468,11 @@ public void run() throws Exception { if (options.reduceAST() && !options.useReducer()) { throw new AssertionError("To reduce AST, use-reducer option must be enabled first"); } - if (reproducer != null && options.useReducer()) { - System.out.println("EXPERIMENTAL: Trying to reduce queries using a simple reducer."); - // System.out.println("Reduced query will be output to stdout but not logs."); + if (options.useReducer()) { + if (reproducer == null) { + logger.getReduceFileWriter().write("current oracle does not support experimental reducer."); + throw new IgnoreMeException(); + } G newGlobalState = createGlobalState(); newGlobalState.setState(stateToRepro); newGlobalState.setRandomly(r); @@ -470,7 +491,14 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } - throw new AssertionError("Found a potential bug"); + try { + logger.getReduceFileWriter().close(); + logger.reduceFileWriter = null; + } catch (IOException e) { + throw new AssertionError(e); + } + + throw new AssertionError("Found a potential bug, please check reducer log for detail."); } } } diff --git a/src/sqlancer/ProviderAdapter.java b/src/sqlancer/ProviderAdapter.java index 1972557d6..346567300 100644 --- a/src/sqlancer/ProviderAdapter.java +++ b/src/sqlancer/ProviderAdapter.java @@ -60,8 +60,7 @@ public Reproducer generateAndTestDatabase(G globalState) throws Exception { try { oracle.check(); globalState.getManager().incrementSelectQueryCount(); - } catch (IgnoreMeException e) { - + } catch (IgnoreMeException ignored) { } catch (AssertionError e) { Reproducer reproducer = oracle.getLastReproducer(); if (reproducer != null) { @@ -69,7 +68,6 @@ public Reproducer generateAndTestDatabase(G globalState) throws Exception { } throw e; } - assert localState != null; localState.executedWithoutError(); } } @@ -141,7 +139,6 @@ public void generateAndTestDatabaseWithQueryPlanGuidance(G globalState) throws E } catch (IgnoreMeException e) { } - assert localState != null; localState.executedWithoutError(); } // exit loop to mutate tables if no new query plans have been found after a while diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 3a5a5c9d0..174efcbb6 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -4,6 +4,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Function; import sqlancer.IgnoreMeException; @@ -38,7 +39,7 @@ public class SQLite3NoRECOracle extends NoRECBase implements private SQLite3ExpressionGenerator gen; private Reproducer reproducer; - private class SQLite3NoRECReproducer implements Reproducer { + private static class SQLite3NoRECReproducer implements Reproducer { private final Function optimizedQuery; private final Function unoptimizedQuery; @@ -50,7 +51,7 @@ private class SQLite3NoRECReproducer implements Reproducer { @Override public boolean bugStillTriggers(SQLite3GlobalState globalState) { - return optimizedQuery.apply(globalState) != unoptimizedQuery.apply(globalState); + return !Objects.equals(optimizedQuery.apply(globalState), unoptimizedQuery.apply(globalState)); } } @@ -119,13 +120,7 @@ private Function getUnoptimizedQuery(SQLite3Select logger.writeCurrent(unoptimizedQueryString); } SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - return new Function() { - - @Override - public Integer apply(SQLite3GlobalState state) { - return extractCounts(q, state); - } - }; + return state -> extractCounts(q, state); } private Function getOptimizedQuery(SQLite3Select select, @@ -147,14 +142,7 @@ private Function getOptimizedQuery(SQLite3Select se logger.writeCurrent(optimizedQueryString); } SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - return new Function() { - - @Override - public Integer apply(SQLite3GlobalState state) { - return useAggregate ? extractCounts(q, state) : countRows(q, state); - } - - }; + return state -> useAggregate ? extractCounts(q, state) : countRows(q, state); } private int countRows(SQLQueryAdapter q, SQLite3GlobalState globalState) { diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java index e03b347f3..f3640800e 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java @@ -6,6 +6,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.visitor.TiDBVisitor; @@ -13,14 +14,50 @@ public class TiDBTLPWhereOracle extends TiDBTLPBase { private String generatedQueryString; + private Reproducer reproducer; public TiDBTLPWhereOracle(TiDBGlobalState state) { super(state); TiDBErrors.addExpressionErrors(errors); } + private class TiDBTLPWhereReproducer implements Reproducer { + final String firstQueryString; + final String secondQueryString; + final String thirdQueryString; + final String originalQueryString; + final List resultSet; + final boolean orderBy; + + TiDBTLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, + String originalQueryString, List resultSet, boolean orderBy) { + this.firstQueryString = firstQueryString; + this.secondQueryString = secondQueryString; + this.thirdQueryString = thirdQueryString; + this.originalQueryString = originalQueryString; + this.resultSet = resultSet; + this.orderBy = orderBy; + } + + @Override + public boolean bugStillTriggers(TiDBGlobalState globalState) { + try { + List combinedString1 = new ArrayList<>(); + List secondResultSet1 = ComparatorHelper.getCombinedResultSet(firstQueryString, + secondQueryString, thirdQueryString, combinedString1, !orderBy, globalState, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet1, originalQueryString, + combinedString1, globalState); + } catch (AssertionError triggeredError) { + return true; + } catch (SQLException ignored) { + } + return false; + } + } + @Override public void check() throws SQLException { + reproducer = null; super.check(); select.setWhereClause(null); String originalQueryString = TiDBVisitor.asString(select); @@ -42,6 +79,8 @@ public void check() throws SQLException { thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, state); + reproducer = new TiDBTLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, + originalQueryString, resultSet, orderBy); } @Override @@ -49,4 +88,9 @@ public String getLastQueryString() { return generatedQueryString; } + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + } From 18f118f7e8cb597bb27acaa07ca6bc8ae183d4a1 Mon Sep 17 00:00:00 2001 From: Akshat Jaimini Date: Wed, 8 Nov 2023 19:37:00 +0530 Subject: [PATCH 395/856] CERT Implementation for PostgreSQL (#896) * Added CERT to Postgres Oracles * Postgres CERT Oracle, added Column Reference * Removed unwanted changes in .settings * Removed .factorypath file * Rows extracted from query plan and query plan successfully extracted * JOIN mutation added * Removed unnecessary code * Restored jdt.core.prefs * Removed unnecessary files * Removed PostgresErrors file --- .../PostgresExpectedValueVisitor.java | 11 + src/sqlancer/postgres/PostgresOptions.java | 12 + .../postgres/PostgresToStringVisitor.java | 12 + src/sqlancer/postgres/PostgresVisitor.java | 10 + .../postgres/ast/PostgresColumnReference.java | 15 + src/sqlancer/postgres/ast/PostgresJoin.java | 73 ++++- .../postgres/ast/PostgresTableReference.java | 15 + src/sqlancer/postgres/gen/PostgresCommon.java | 1 + .../postgres/oracle/PostgresCERTOracle.java | 266 ++++++++++++++++++ 9 files changed, 413 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/postgres/ast/PostgresColumnReference.java create mode 100644 src/sqlancer/postgres/ast/PostgresTableReference.java create mode 100644 src/sqlancer/postgres/oracle/PostgresCERTOracle.java diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index d75aa4a5e..efc1b7442 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -5,6 +5,7 @@ import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; +import sqlancer.postgres.ast.PostgresColumnReference; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; @@ -20,6 +21,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -75,6 +77,15 @@ public void visit(PostgresPrefixOperation op) { visit(op.getExpression()); } + @Override + public void visit(PostgresColumnReference column) { + print(column); + } + + @Override + public void visit(PostgresTableReference tb) { + } + @Override public void visit(PostgresSelect op) { visit(op.getWhereClause()); diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index bb76404de..ced65d5da 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -13,6 +13,7 @@ import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; +import sqlancer.postgres.oracle.PostgresCERTOracle; import sqlancer.postgres.oracle.PostgresFuzzer; import sqlancer.postgres.oracle.PostgresNoRECOracle; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; @@ -78,6 +79,17 @@ public TestOracle create(PostgresGlobalState globalState) t return new CompositeTestOracle(oracles, globalState); } }, + CERT { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, FUZZER { @Override public TestOracle create(PostgresGlobalState globalState) throws Exception { diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 1e95ae745..d1766e6c8 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -11,6 +11,7 @@ import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; +import sqlancer.postgres.ast.PostgresColumnReference; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; @@ -28,6 +29,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; public final class PostgresToStringVisitor extends ToStringVisitor implements PostgresVisitor { @@ -46,6 +48,11 @@ public String get() { return sb.toString(); } + @Override + public void visit(PostgresColumnReference column) { + sb.append(column.getColumn().getFullQualifiedName()); + } + @Override public void visit(PostgresPostfixOperation op) { sb.append("("); @@ -87,6 +94,11 @@ public void visit(PostgresSubquery subquery) { sb.append(subquery.getName()); } + @Override + public void visit(PostgresTableReference ref) { + sb.append(ref.getTable().getName()); + } + @Override public void visit(PostgresSelect s) { sb.append("SELECT "); diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index d66dd1271..d9c82232c 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -9,6 +9,7 @@ import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; import sqlancer.postgres.ast.PostgresCastOperation; import sqlancer.postgres.ast.PostgresCollate; +import sqlancer.postgres.ast.PostgresColumnReference; import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; @@ -24,6 +25,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; import sqlancer.postgres.gen.PostgresExpressionGenerator; public interface PostgresVisitor { @@ -34,6 +36,10 @@ public interface PostgresVisitor { void visit(PostgresColumnValue c); + void visit(PostgresColumnReference c); + + void visit(PostgresTableReference tb); + void visit(PostgresPrefixOperation op); void visit(PostgresSelect op); @@ -103,6 +109,10 @@ default void visit(PostgresExpression expression) { visit((PostgresSubquery) expression); } else if (expression instanceof PostgresLikeOperation) { visit((PostgresLikeOperation) expression); + } else if (expression instanceof PostgresColumnReference) { + visit((PostgresColumnReference) expression); + } else if (expression instanceof PostgresTableReference) { + visit((PostgresTableReference) expression); } else { throw new AssertionError(expression); } diff --git a/src/sqlancer/postgres/ast/PostgresColumnReference.java b/src/sqlancer/postgres/ast/PostgresColumnReference.java new file mode 100644 index 000000000..0d835bf89 --- /dev/null +++ b/src/sqlancer/postgres/ast/PostgresColumnReference.java @@ -0,0 +1,15 @@ +package sqlancer.postgres.ast; + +import sqlancer.postgres.PostgresSchema.PostgresColumn; + +public class PostgresColumnReference implements PostgresExpression { + private final PostgresColumn c; + + public PostgresColumnReference(PostgresColumn c) { + this.c = c; + } + + public PostgresColumn getColumn() { + return c; + } +} diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index 57b92ddf3..d1a076bbe 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -1,7 +1,14 @@ package sqlancer.postgres.ast; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import sqlancer.Randomly; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.gen.PostgresExpressionGenerator; public class PostgresJoin implements PostgresExpression { @@ -12,22 +19,84 @@ public static PostgresJoinType getRandom() { return Randomly.fromOptions(values()); } + public static PostgresJoinType getRandomExcept(PostgresJoinType... exclude) { + PostgresJoinType[] values = Arrays.stream(values()).filter(m -> !Arrays.asList(exclude).contains(m)) + .toArray(PostgresJoinType[]::new); + return Randomly.fromOptions(values); + } + } private final PostgresExpression tableReference; - private final PostgresExpression onClause; - private final PostgresJoinType type; + private PostgresExpression onClause; + private PostgresJoinType type; + private final PostgresExpression leftTable; + private final PostgresExpression rightTable; public PostgresJoin(PostgresExpression tableReference, PostgresExpression onClause, PostgresJoinType type) { this.tableReference = tableReference; this.onClause = onClause; this.type = type; + this.leftTable = null; + this.rightTable = null; + } + + public PostgresJoin(PostgresExpression leftTable, PostgresExpression rightTable, PostgresJoinType joinType, + PostgresExpression whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.type = joinType; + this.onClause = whereCondition; + this.tableReference = null; + } + + public static PostgresJoin createJoin(PostgresExpression left, PostgresExpression right, PostgresJoinType type, + PostgresExpression onClause) { + if (type == PostgresJoinType.CROSS) { + return new PostgresJoin(left, right, type, null); + } else { + return new PostgresJoin(left, right, type, onClause); + } + } + + public static List getJoins(List tableList, + PostgresGlobalState globalState) { + // Clone Table to prevent the original list from being manipulated + List tbl = new ArrayList<>(tableList); + List joinExpressions = new ArrayList<>(); + while (tbl.size() >= 2 && Randomly.getBoolean()) { + PostgresTableReference left = (PostgresTableReference) tbl.remove(0); + PostgresTableReference right = (PostgresTableReference) tbl.remove(0); + List columns = new ArrayList<>(); + columns.addAll(left.getTable().getColumns()); + columns.addAll(right.getTable().getColumns()); + PostgresExpressionGenerator joinGen = new PostgresExpressionGenerator(globalState).setColumns(columns); + joinExpressions.add(PostgresJoin.createJoin(left, right, PostgresJoinType.getRandom(), + joinGen.generateExpression(0, PostgresDataType.BOOLEAN))); + } + return joinExpressions; + } + + public void setOnClause(PostgresExpression clause) { + this.onClause = clause; + } + + public void setType(PostgresJoinType type) { + this.type = type; } public PostgresExpression getTableReference() { return tableReference; } + public PostgresExpression getLeftTable() { + return leftTable; + } + + public PostgresExpression getRightTable() { + return rightTable; + } + public PostgresExpression getOnClause() { return onClause; } diff --git a/src/sqlancer/postgres/ast/PostgresTableReference.java b/src/sqlancer/postgres/ast/PostgresTableReference.java new file mode 100644 index 000000000..2abf8a7d2 --- /dev/null +++ b/src/sqlancer/postgres/ast/PostgresTableReference.java @@ -0,0 +1,15 @@ +package sqlancer.postgres.ast; + +import sqlancer.postgres.PostgresSchema.PostgresTable; + +public class PostgresTableReference implements PostgresExpression { + private final PostgresTable table; + + public PostgresTableReference(PostgresTable table) { + this.table = table; + } + + public PostgresTable getTable() { + return table; + } +} diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 99bb7fa66..497ad6d0b 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -134,6 +134,7 @@ public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { errors.add("value too long for type character"); + errors.add("cannot insert a non-DEFAULT value into column"); errors.add("not found in view targetlist"); } diff --git a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java new file mode 100644 index 000000000..9b509834e --- /dev/null +++ b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java @@ -0,0 +1,266 @@ +package sqlancer.postgres.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.oracle.CERTOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTables; +import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; +import sqlancer.postgres.ast.PostgresBinaryLogicalOperation.BinaryLogicalOperator; +import sqlancer.postgres.ast.PostgresColumnReference; +import sqlancer.postgres.ast.PostgresConstant; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; + +public class PostgresCERTOracle extends CERTOracleBase implements TestOracle { + private PostgresExpressionGenerator gen; + private PostgresSelect select; + + public PostgresCERTOracle(PostgresGlobalState globalState) { + super(globalState); + PostgresCommon.addCommonExpressionErrors(errors); + PostgresCommon.addCommonInsertUpdateErrors(errors); + PostgresCommon.addGroupingErrors(errors); + PostgresCommon.addCommonInsertUpdateErrors(errors); + PostgresCommon.addCommonRangeExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + queryPlan1Sequences = new ArrayList<>(); + queryPlan2Sequences = new ArrayList<>(); + + // Generate Random Query + PostgresTables tables = state.getSchema().getRandomTableNonEmptyTables(); + List tableList = tables.getTables().stream().map(t -> new PostgresTableReference(t)) + .collect(Collectors.toList()); + gen = new PostgresExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new PostgresColumnReference(c)).collect(Collectors.toList())); + + select = new PostgresSelect(); + select.setFetchColumns(fetchColumns); + select.setFromList(tableList); + List joins = PostgresJoin.getJoins(tableList, state); + select.setJoinList(joins); + + select.setSelectType(Randomly.fromOptions(PostgresSelect.SelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + } + + // First query row count + String queryString1 = PostgresVisitor.asString(select); + int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + // JOIN and LIMIT mutations not added + boolean increase = mutate(Mutator.LIMIT); + + // Second Query row count + String queryString2 = PostgresVisitor.asString(select); + int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + // Check query plan equivalence + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check results + if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { + throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 + + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); + } + } + + @Override + protected boolean mutateJoin() { + if (select.getJoinList().isEmpty()) { + return false; + } + PostgresJoin join = (PostgresJoin) Randomly.fromList(select.getJoinList()); + + // Exclude CROSS for on condition + if (join.getType() == PostgresJoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((PostgresTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((PostgresTableReference) join.getRightTable()).getTable().getColumns()); + PostgresExpressionGenerator joinGen2 = new PostgresExpressionGenerator(state).setColumns(columns); + join.setOnClause(joinGen2.generateExpression(0, PostgresDataType.BOOLEAN)); + } + + PostgresJoinType newJoinType = PostgresJoinType.INNER; + if (join.getType() == PostgresJoinType.LEFT || join.getType() == PostgresJoinType.RIGHT) { + newJoinType = PostgresJoinType.getRandomExcept(PostgresJoinType.LEFT, PostgresJoinType.RIGHT); + } else { + newJoinType = PostgresJoinType.getRandomExcept(join.getType()); + } + boolean increase = join.getType().ordinal() < newJoinType.ordinal(); + join.setType(newJoinType); + if (newJoinType == PostgresJoinType.CROSS) { + join.setOnClause(null); + } + return increase; + } + + @Override + protected boolean mutateDistinct() { + PostgresSelect.SelectType selectType = select.getSelectOption(); + if (selectType != PostgresSelect.SelectType.ALL) { + select.setSelectType(PostgresSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(PostgresSelect.SelectType.DISTINCT); + return false; + } + } + + @Override + protected boolean mutateWhere() { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + return increase; + } + + @Override + protected boolean mutateGroupBy() { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + @Override + protected boolean mutateHaving() { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + @Override + protected boolean mutateAnd() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + @Override + protected boolean mutateOr() { + if (select.getWhereClause() == null) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + gen.generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + @Override + protected boolean mutateLimit() { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + Randomly r = new Randomly(); + select.setLimitClause(PostgresConstant.createIntConstant((int) Math.abs(r.getInteger()))); + } + return increase; + } + + private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + throws AssertionError, SQLException { + int row = -1; + String explainQuery = "EXPLAIN " + selectStr; + + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + String content = rs.getString(1).trim(); + if (content.contains("rows=")) { + try { + int ind = content.indexOf("rows="); + int number = Integer.parseInt(content.substring(ind + 5).split(" ")[0]); + if (row == -1) { + row = number; + + } + } catch (Exception e) { + } + } + // Proper Formatting TBD + String[] planPart = content.split("-> "); + String plan = planPart[planPart.length - 1]; + queryPlanSequences.add(plan.split(" ")[0].trim()); + } + + } + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + if (row == -1) { + throw new IgnoreMeException(); + } + return row; + + } +} From 201fd4c24aba5378447dc6b521e86d3b85ee485f Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Thu, 30 Nov 2023 14:18:34 +0100 Subject: [PATCH 396/856] Fix query plan collection for TiDB (#900) For `EXPLAIN`, the output includes the random id for each operator, such as `IndexLookUp_10`. While, according to the technical paper of QPG, the random id should be removed. Therefore, we fix this issue by using `EXPLAIN FORMAT=simplified`. --- src/sqlancer/tidb/TiDBProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 3e4133761..c5c334f11 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -208,7 +208,7 @@ public String getQueryPlan(String selectStr, TiDBGlobalState globalState) throws } } - SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN " + selectStr); + SQLQueryAdapter q = new SQLQueryAdapter("EXPLAIN FORMAT=brief " + selectStr); try (SQLancerResultSet rs = q.executeAndGet(globalState)) { if (rs != null) { while (rs.next()) { From 24a42644b0fd03fa8e4b5476b6a20c4afcd1a445 Mon Sep 17 00:00:00 2001 From: Marcus Gartner <1128750+mgartner@users.noreply.github.com> Date: Wed, 13 Dec 2023 00:05:54 -0800 Subject: [PATCH 397/856] Upgrade CockroachDB to v23.2.0-beta.1 (#902) --- .github/workflows/main.yml | 14 +++++++------- .../gen/CockroachDBSetClusterSettingGenerator.java | 3 ++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 612c6f571..af92103d7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -134,11 +134,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-beta.2.linux-amd64.tgz | tar xvz - cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.2.0-beta.1.linux-amd64.tgz | tar xvz + cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -159,11 +159,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.1.0-beta.2.linux-amd64.tgz | tar xvz - cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v23.2.0-beta.1.linux-amd64.tgz | tar xvz + cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.1.0-beta.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test @@ -567,4 +567,4 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - mvn -Dtest=TestStatementReducer test \ No newline at end of file + mvn -Dtest=TestStatementReducer test diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java index cfc89562a..a948a1237 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBSetClusterSettingGenerator.java @@ -17,7 +17,8 @@ private CockroachDBSetClusterSettingGenerator() { private enum CockroachDBClusterSetting { BACKPRESSURE_RANGE_SIZE_MULTIPLIER(" kv.range.backpressure_range_size_multiplier", (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), - RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", (g) -> Randomly.getNonCachedInteger()), + RANGE_DESCRIPTOR_CACHE_SIZE("kv.range_descriptor_cache.size", + (g) -> Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)), SQL_QUERY_CACHE_ENABLED("sql.query_cache.enabled", CockroachDBSetSessionGenerator::onOff), SQL_STATS_HISTOGRAM_COLLECTION_ENABLED("sql.stats.histogram_collection.enabled", CockroachDBSetSessionGenerator::onOff), From 228dd53a75abe7f436fbe6090d53b3765a1459e0 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sat, 30 Dec 2023 07:29:19 +0100 Subject: [PATCH 398/856] Update QPG paper link (#903) --- docs/PAPERS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/PAPERS.md b/docs/PAPERS.md index 6510930e1..c0e13dd3e 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -53,7 +53,7 @@ This paper describes TLP, a metamorphic testing approach that can detect various ## Testing Database Engines via Query Plan Guidance -This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. A preprint is available [here](http://jinshengba.me/assets/pdf/qpg_icse23.pdf). +This paper describes Query Plan Guidance (QPG), a test case generation method guided by query plan coverage. This method can be paired with above three testing methods. A preprint is available [here](http://bajinsheng.github.io/assets/pdf/qpg_icse23.pdf). ``` @inproceedings{Ba2023QPG, From 0d6ba67c92d2a844fb9c4adb713655f9e5934ae1 Mon Sep 17 00:00:00 2001 From: Aryan Patil <95161886+AryanP45@users.noreply.github.com> Date: Mon, 8 Jan 2024 22:00:48 +0530 Subject: [PATCH 399/856] Typo in testH2 (#905) --- test/sqlancer/dbms/TestH2.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/sqlancer/dbms/TestH2.java b/test/sqlancer/dbms/TestH2.java index 8f13170ff..701f6d50b 100644 --- a/test/sqlancer/dbms/TestH2.java +++ b/test/sqlancer/dbms/TestH2.java @@ -10,10 +10,10 @@ public class TestH2 { @Test - public void testDuckDB() { + public void testH2DB() { String h2Available = System.getenv("H2_AVAILABLE"); - boolean mariaDBIsAvailable = h2Available != null && h2Available.equalsIgnoreCase("true"); - assumeTrue(mariaDBIsAvailable); + boolean h2DBIsAvailable = h2Available != null && h2Available.equalsIgnoreCase("true"); + assumeTrue(h2DBIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "h2" })); From 3e960fb16fac42ed8f43eecf25eb4d09a5be9d85 Mon Sep 17 00:00:00 2001 From: Robins Date: Wed, 17 Jan 2024 03:18:01 +1030 Subject: [PATCH 400/856] Make --test-collations=false more robust. (#908) * Make --test-collations=false more robust. With --test-collations=false, database creation would now enforce using UTF8 and template0 database. This ensures that postgres clusters created in other locales still try to create UTF8 databases and test other aspects of the database engine. No change made to scenarios where --test-collations=true. * Conflate default CREATE DATABASE options into single string if --test-collations=false. --- src/sqlancer/postgres/PostgresProvider.java | 22 ++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index e9396a628..62c43cd5a 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -325,18 +325,22 @@ protected void prepareTables(PostgresGlobalState globalState) throws Exception { private String getCreateDatabaseCommand(PostgresGlobalState state) { StringBuilder sb = new StringBuilder(); sb.append("CREATE DATABASE " + databaseName + " "); - if (Randomly.getBoolean() && ((PostgresOptions) state.getDbmsSpecificOptions()).testCollations) { + if (((PostgresOptions) state.getDbmsSpecificOptions()).testCollations) { if (Randomly.getBoolean()) { - sb.append("WITH ENCODING '"); - sb.append(Randomly.fromOptions("utf8")); - sb.append("' "); - } - for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { - if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { - sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + if (Randomly.getBoolean()) { + sb.append("WITH ENCODING '"); + sb.append(Randomly.fromOptions("utf8")); + sb.append("' "); + } + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } } + sb.append(" TEMPLATE template0"); } - sb.append(" TEMPLATE template0"); + } else { + sb.append("WITH ENCODING 'UTF8' TEMPLATE template0"); } return sb.toString(); } From 5560b72e17dbb0f7817d0afd1f200884daaaebed Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Thu, 22 Feb 2024 20:26:25 +0800 Subject: [PATCH 401/856] Update options to be of the same type (#911) Fixes the build error: - java: The method fromOptions(T...) of type sqlancer.Randomly is not applicable as the formal varargs element type T is not accessible here --- src/sqlancer/mariadb/gen/MariaDBSetGenerator.java | 6 +++--- src/sqlancer/mysql/gen/MySQLAlterTable.java | 6 +++--- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index d437d4537..f4caf0ec6 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -35,11 +35,11 @@ private enum Action { AUTOCOMMIT("autocommit", (r) -> 1, Scope.GLOBAL, Scope.SESSION), // BIG_TABLES("big_tables", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - COMPLETION_TYPE("completion_type", (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", 0, 1, 2), - Scope.GLOBAL), // + COMPLETION_TYPE("completion_type", + (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", "0", "1", "2"), Scope.GLOBAL), // // BULK_INSERT_CACHE_SIZE("bulk_insert_buffer_size", (r) -> r.getLong(0, Long.MAX_VALUE), Scope.GLOBAL, // Scope.SESSION), - CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", 0, 1, 2), + CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", "0", "1", "2"), Scope.GLOBAL), CTE_MAX_RECURSION_DEPTH("cte_max_recursion_depth", (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), DELAY_KEY_WRITE("delay_key_write", (r) -> Randomly.fromOptions("ON", "OFF", "ALL"), Scope.GLOBAL), diff --git a/src/sqlancer/mysql/gen/MySQLAlterTable.java b/src/sqlancer/mysql/gen/MySQLAlterTable.java index cb5db6bde..f2c952016 100644 --- a/src/sqlancer/mysql/gen/MySQLAlterTable.java +++ b/src/sqlancer/mysql/gen/MySQLAlterTable.java @@ -130,15 +130,15 @@ private SQLQueryAdapter create() { break; case STATS_AUTO_RECALC: sb.append("STATS_AUTO_RECALC "); - sb.append(Randomly.fromOptions(0, 1, "DEFAULT")); + sb.append(Randomly.fromOptions("0", "1", "DEFAULT")); break; case STATS_PERSISTENT: sb.append("STATS_PERSISTENT "); - sb.append(Randomly.fromOptions(0, 1, "DEFAULT")); + sb.append(Randomly.fromOptions("0", "1", "DEFAULT")); break; case PACK_KEYS: sb.append("PACK_KEYS "); - sb.append(Randomly.fromOptions(0, 1, "DEFAULT")); + sb.append(Randomly.fromOptions("0", "1", "DEFAULT")); break; // not relevant: // case WITH_WITHOUT_VALIDATION: diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 546962fe6..821184617 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -34,11 +34,11 @@ private enum Action { AUTOCOMMIT("autocommit", (r) -> 1, Scope.GLOBAL, Scope.SESSION), // BIG_TABLES("big_tables", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - COMPLETION_TYPE("completion_type", (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", 0, 1, 2), - Scope.GLOBAL), // + COMPLETION_TYPE("completion_type", + (r) -> Randomly.fromOptions("'NO_CHAIN'", "'CHAIN'", "'RELEASE'", "0", "1", "2"), Scope.GLOBAL), // BULK_INSERT_CACHE_SIZE("bulk_insert_buffer_size", (r) -> r.getLong(0, Long.MAX_VALUE), Scope.GLOBAL, // Scope.SESSION), // - CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", 0, 1, 2), // + CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", "0", "1", "2"), // Scope.GLOBAL), // CTE_MAX_RECURSION_DEPTH("cte_max_recursion_depth", // (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), // From ece708d2c80150faa320f81cf203da7f93b89150 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Sat, 9 Mar 2024 09:27:45 +0800 Subject: [PATCH 402/856] Remove NoSQL database (#915) --- src/check_names.py | 2 - src/sqlancer/Main.java | 6 - .../arangodb/ArangoDBComparatorHelper.java | 73 ----- src/sqlancer/arangodb/ArangoDBConnection.java | 31 -- .../arangodb/ArangoDBLoggableFactory.java | 40 --- src/sqlancer/arangodb/ArangoDBOptions.java | 50 --- src/sqlancer/arangodb/ArangoDBProvider.java | 137 --------- .../arangodb/ArangoDBQueryAdapter.java | 16 - .../arangodb/ArangoDBQueryProvider.java | 6 - src/sqlancer/arangodb/ArangoDBSchema.java | 70 ----- .../arangodb/ast/ArangoDBConstant.java | 108 ------- .../arangodb/ast/ArangoDBExpression.java | 4 - src/sqlancer/arangodb/ast/ArangoDBSelect.java | 79 ----- .../ast/ArangoDBUnsupportedPredicate.java | 6 - .../ArangoDBComputedExpressionGenerator.java | 85 ----- .../gen/ArangoDBCreateIndexGenerator.java | 18 -- .../ArangoDBFilterExpressionGenerator.java | 153 --------- .../arangodb/gen/ArangoDBInsertGenerator.java | 39 --- .../arangodb/gen/ArangoDBTableGenerator.java | 44 --- .../query/ArangoDBConstantGenerator.java | 46 --- .../query/ArangoDBCreateIndexQuery.java | 54 ---- .../query/ArangoDBCreateTableQuery.java | 44 --- .../arangodb/query/ArangoDBInsertQuery.java | 66 ---- .../query/ArangoDBOptimizerRules.java | 57 ---- .../arangodb/query/ArangoDBSelectQuery.java | 88 ------ .../test/ArangoDBQueryPartitioningBase.java | 67 ---- .../ArangoDBQueryPartitioningWhereTester.java | 46 --- .../visitor/ArangoDBToQueryVisitor.java | 134 -------- .../arangodb/visitor/ArangoDBVisitor.java | 51 --- src/sqlancer/cosmos/CosmosProvider.java | 77 ----- .../mongodb/MongoDBComparatorHelper.java | 97 ------ src/sqlancer/mongodb/MongoDBConnection.java | 35 --- .../mongodb/MongoDBLoggableFactory.java | 40 --- src/sqlancer/mongodb/MongoDBOptions.java | 74 ----- src/sqlancer/mongodb/MongoDBProvider.java | 128 -------- src/sqlancer/mongodb/MongoDBQueryAdapter.java | 15 - .../mongodb/MongoDBQueryProvider.java | 6 - src/sqlancer/mongodb/MongoDBSchema.java | 97 ------ .../ast/MongoDBBinaryComparisonNode.java | 16 - .../mongodb/ast/MongoDBBinaryLogicalNode.java | 16 - src/sqlancer/mongodb/ast/MongoDBConstant.java | 252 --------------- .../mongodb/ast/MongoDBExpression.java | 4 - .../mongodb/ast/MongoDBRegexNode.java | 24 -- src/sqlancer/mongodb/ast/MongoDBSelect.java | 104 ------- .../ast/MongoDBUnaryLogicalOperatorNode.java | 16 - .../ast/MongoDBUnsupportedPredicate.java | 7 - .../MongoDBComputedExpressionGenerator.java | 89 ------ .../mongodb/gen/MongoDBConstantGenerator.java | 86 ------ .../mongodb/gen/MongoDBIndexGenerator.java | 25 -- .../mongodb/gen/MongoDBInsertGenerator.java | 38 --- .../gen/MongoDBMatchExpressionGenerator.java | 291 ------------------ .../mongodb/gen/MongoDBTableGenerator.java | 54 ---- .../query/MongoDBCreateIndexQuery.java | 77 ----- .../query/MongoDBCreateTableQuery.java | 115 ------- .../mongodb/query/MongoDBInsertQuery.java | 87 ------ .../mongodb/query/MongoDBRemoveQuery.java | 59 ---- .../mongodb/query/MongoDBSelectQuery.java | 147 --------- .../test/MongoDBColumnTestReference.java | 40 --- .../test/MongoDBDocumentRemovalBase.java | 90 ------ .../test/MongoDBDocumentRemovalTester.java | 49 --- .../test/MongoDBQueryPartitioningBase.java | 93 ------ .../MongoDBQueryPartitioningWhereTester.java | 48 --- .../mongodb/visitor/MongoDBNegateVisitor.java | 161 ---------- .../mongodb/visitor/MongoDBToLogVisitor.java | 195 ------------ .../visitor/MongoDBToQueryVisitor.java | 184 ----------- .../mongodb/visitor/MongoDBVisitor.java | 45 --- 66 files changed, 4601 deletions(-) delete mode 100644 src/sqlancer/arangodb/ArangoDBComparatorHelper.java delete mode 100644 src/sqlancer/arangodb/ArangoDBConnection.java delete mode 100644 src/sqlancer/arangodb/ArangoDBLoggableFactory.java delete mode 100644 src/sqlancer/arangodb/ArangoDBOptions.java delete mode 100644 src/sqlancer/arangodb/ArangoDBProvider.java delete mode 100644 src/sqlancer/arangodb/ArangoDBQueryAdapter.java delete mode 100644 src/sqlancer/arangodb/ArangoDBQueryProvider.java delete mode 100644 src/sqlancer/arangodb/ArangoDBSchema.java delete mode 100644 src/sqlancer/arangodb/ast/ArangoDBConstant.java delete mode 100644 src/sqlancer/arangodb/ast/ArangoDBExpression.java delete mode 100644 src/sqlancer/arangodb/ast/ArangoDBSelect.java delete mode 100644 src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java delete mode 100644 src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBInsertQuery.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java delete mode 100644 src/sqlancer/arangodb/query/ArangoDBSelectQuery.java delete mode 100644 src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java delete mode 100644 src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java delete mode 100644 src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java delete mode 100644 src/sqlancer/arangodb/visitor/ArangoDBVisitor.java delete mode 100644 src/sqlancer/cosmos/CosmosProvider.java delete mode 100644 src/sqlancer/mongodb/MongoDBComparatorHelper.java delete mode 100644 src/sqlancer/mongodb/MongoDBConnection.java delete mode 100644 src/sqlancer/mongodb/MongoDBLoggableFactory.java delete mode 100644 src/sqlancer/mongodb/MongoDBOptions.java delete mode 100644 src/sqlancer/mongodb/MongoDBProvider.java delete mode 100644 src/sqlancer/mongodb/MongoDBQueryAdapter.java delete mode 100644 src/sqlancer/mongodb/MongoDBQueryProvider.java delete mode 100644 src/sqlancer/mongodb/MongoDBSchema.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBConstant.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBExpression.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBRegexNode.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBSelect.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java delete mode 100644 src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java delete mode 100644 src/sqlancer/mongodb/gen/MongoDBTableGenerator.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBInsertQuery.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBRemoveQuery.java delete mode 100644 src/sqlancer/mongodb/query/MongoDBSelectQuery.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBColumnTestReference.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java delete mode 100644 src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java delete mode 100644 src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java delete mode 100644 src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java delete mode 100644 src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java delete mode 100644 src/sqlancer/mongodb/visitor/MongoDBVisitor.java diff --git a/src/check_names.py b/src/check_names.py index 048da7669..63c1f093f 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -33,12 +33,10 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): cwd = os.getcwd() print("Current working directory: {0}".format(cwd)) name_to_files: dict[str:List[str]] = dict() - name_to_files["ArangoDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "arangodb")) name_to_files["Citus"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "citus")) name_to_files["ClickHouse"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "clickhouse")) name_to_files["CnosDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cnosdb")) name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) - name_to_files["Cosmos"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cosmos")) name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) name_to_files["DuckDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "duckdb")) name_to_files["H2"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "h2")) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index ebb3f1213..1cbc02645 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -23,7 +23,6 @@ import com.beust.jcommander.JCommander; import com.beust.jcommander.JCommander.Builder; -import sqlancer.arangodb.ArangoDBProvider; import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; import sqlancer.cnosdb.CnosDBProvider; @@ -31,7 +30,6 @@ import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.cosmos.CosmosProvider; import sqlancer.databend.DatabendProvider; import sqlancer.doris.DorisProvider; import sqlancer.duckdb.DuckDBProvider; @@ -39,7 +37,6 @@ import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.mariadb.MariaDBProvider; import sqlancer.materialize.MaterializeProvider; -import sqlancer.mongodb.MongoDBProvider; import sqlancer.mysql.MySQLProvider; import sqlancer.oceanbase.OceanBaseProvider; import sqlancer.postgres.PostgresProvider; @@ -732,12 +729,10 @@ private static void checkForIssue799(List> providers) if (providers.isEmpty()) { System.err.println( "No DBMS implementations (i.e., instantiations of the DatabaseProvider class) were found. You likely ran into an issue described in https://github.com/sqlancer/sqlancer/issues/799. As a workaround, I now statically load all supported providers as of June 7, 2023."); - providers.add(new ArangoDBProvider()); providers.add(new CitusProvider()); providers.add(new ClickHouseProvider()); providers.add(new CnosDBProvider()); providers.add(new CockroachDBProvider()); - providers.add(new CosmosProvider()); providers.add(new DatabendProvider()); providers.add(new DorisProvider()); providers.add(new DuckDBProvider()); @@ -745,7 +740,6 @@ private static void checkForIssue799(List> providers) providers.add(new HSQLDBProvider()); providers.add(new MariaDBProvider()); providers.add(new MaterializeProvider()); - providers.add(new MongoDBProvider()); providers.add(new MySQLProvider()); providers.add(new OceanBaseProvider()); providers.add(new PrestoProvider()); diff --git a/src/sqlancer/arangodb/ArangoDBComparatorHelper.java b/src/sqlancer/arangodb/ArangoDBComparatorHelper.java deleted file mode 100644 index 2a00a312d..000000000 --- a/src/sqlancer/arangodb/ArangoDBComparatorHelper.java +++ /dev/null @@ -1,73 +0,0 @@ -package sqlancer.arangodb; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.IgnoreMeException; -import sqlancer.Main; -import sqlancer.arangodb.query.ArangoDBSelectQuery; -import sqlancer.common.query.ExpectedErrors; - -public final class ArangoDBComparatorHelper { - - private ArangoDBComparatorHelper() { - - } - - public static List getResultSetAsDocumentList(ArangoDBSelectQuery query, - ArangoDBProvider.ArangoDBGlobalState state) throws Exception { - ExpectedErrors errors = query.getExpectedErrors(); - List result; - try { - query.executeAndGet(state); - Main.nrSuccessfulActions.addAndGet(1); - result = query.getResultSet(); - return result; - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw e; - } - Main.nrUnsuccessfulActions.addAndGet(1); - if (e.getMessage() == null) { - throw new AssertionError(query.getLogString(), e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(query.getLogString(), e); - } - - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - ArangoDBSelectQuery originalQuery) { - if (resultSet.size() != secondResultSet.size()) { - String assertionMessage = String.format("The Size of the result sets mismatch (%d and %d)!\n%s", - resultSet.size(), secondResultSet.size(), originalQuery.getLogString()); - throw new AssertionError(assertionMessage); - } - Set firstHashSet = new HashSet<>(resultSet); - Set secondHashSet = new HashSet<>(secondResultSet); - - if (!firstHashSet.equals(secondHashSet)) { - Set firstResultSetMisses = new HashSet<>(firstHashSet); - firstResultSetMisses.removeAll(secondHashSet); - Set secondResultSetMisses = new HashSet<>(secondHashSet); - secondResultSetMisses.removeAll(firstHashSet); - StringBuilder firstMisses = new StringBuilder(); - for (BaseDocument document : firstResultSetMisses) { - firstMisses.append(document).append(" "); - } - StringBuilder secondMisses = new StringBuilder(); - for (BaseDocument document : secondResultSetMisses) { - secondMisses.append(document).append(" "); - } - String assertMessage = String.format("The Content of the result sets mismatch!\n %s \n %s\n %s", - firstMisses.toString(), secondMisses.toString(), originalQuery.getLogString()); - throw new AssertionError(assertMessage); - } - } -} diff --git a/src/sqlancer/arangodb/ArangoDBConnection.java b/src/sqlancer/arangodb/ArangoDBConnection.java deleted file mode 100644 index b3e5b85d3..000000000 --- a/src/sqlancer/arangodb/ArangoDBConnection.java +++ /dev/null @@ -1,31 +0,0 @@ -package sqlancer.arangodb; - -import com.arangodb.ArangoDB; -import com.arangodb.ArangoDatabase; - -import sqlancer.SQLancerDBConnection; - -public class ArangoDBConnection implements SQLancerDBConnection { - - private final ArangoDB client; - private final ArangoDatabase database; - - public ArangoDBConnection(ArangoDB client, ArangoDatabase database) { - this.client = client; - this.database = database; - } - - @Override - public String getDatabaseVersion() throws Exception { - return client.getVersion().getVersion(); - } - - @Override - public void close() throws Exception { - client.shutdown(); - } - - public ArangoDatabase getDatabase() { - return database; - } -} diff --git a/src/sqlancer/arangodb/ArangoDBLoggableFactory.java b/src/sqlancer/arangodb/ArangoDBLoggableFactory.java deleted file mode 100644 index 927d9f320..000000000 --- a/src/sqlancer/arangodb/ArangoDBLoggableFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -package sqlancer.arangodb; - -import java.util.Arrays; - -import sqlancer.common.log.Loggable; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.log.LoggedString; -import sqlancer.common.query.Query; - -public class ArangoDBLoggableFactory extends LoggableFactory { - @Override - protected Loggable createLoggable(String input, String suffix) { - return new LoggedString(input + suffix); - } - - @Override - public Query getQueryForStateToReproduce(String queryString) { - throw new UnsupportedOperationException(); - } - - @Override - public Query commentOutQuery(Query query) { - throw new UnsupportedOperationException(); - } - - @Override - protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { - StringBuilder sb = new StringBuilder(); - sb.append("// Time: ").append(time).append("\n"); - sb.append("// Database: ").append(databaseName).append("\n"); - sb.append("// Database version: ").append(databaseVersion).append("\n"); - sb.append("// seed value: ").append(seedValue).append("\n"); - return new LoggedString(sb.toString()); - } - - @Override - public Loggable convertStacktraceToLoggable(Throwable throwable) { - return new LoggedString(Arrays.toString(throwable.getStackTrace()) + "\n" + throwable.getMessage()); - } -} diff --git a/src/sqlancer/arangodb/ArangoDBOptions.java b/src/sqlancer/arangodb/ArangoDBOptions.java deleted file mode 100644 index 7e20883d5..000000000 --- a/src/sqlancer/arangodb/ArangoDBOptions.java +++ /dev/null @@ -1,50 +0,0 @@ -package sqlancer.arangodb; - -import static sqlancer.arangodb.ArangoDBOptions.ArangoDBOracleFactory.QUERY_PARTITIONING; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.arangodb.ArangoDBProvider.ArangoDBGlobalState; -import sqlancer.arangodb.test.ArangoDBQueryPartitioningWhereTester; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; - -@Parameters(commandDescription = "ArangoDB (experimental)") -public class ArangoDBOptions implements DBMSSpecificOptions { - - @Parameter(names = "--oracle") - public List oracles = Arrays.asList(QUERY_PARTITIONING); - - @Parameter(names = "--test-random-type-inserts", description = "Insert random types instead of schema types.") - public boolean testRandomTypeInserts; - - @Parameter(names = "--max-number-indexes", description = "The maximum number of indexes used.", arity = 1) - public int maxNumberIndexes = 15; - - @Parameter(names = "--with-optimizer-rule-tests", description = "Adds an additional query, where a random set" - + "of optimizer rules are disabled.", arity = 1) - public boolean withOptimizerRuleTests; - - @Override - public List getTestOracleFactory() { - return oracles; - } - - public enum ArangoDBOracleFactory implements OracleFactory { - QUERY_PARTITIONING { - @Override - public TestOracle create(ArangoDBGlobalState globalState) throws Exception { - List> oracles = new ArrayList<>(); - oracles.add(new ArangoDBQueryPartitioningWhereTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - } - } -} diff --git a/src/sqlancer/arangodb/ArangoDBProvider.java b/src/sqlancer/arangodb/ArangoDBProvider.java deleted file mode 100644 index 7bdc2fa01..000000000 --- a/src/sqlancer/arangodb/ArangoDBProvider.java +++ /dev/null @@ -1,137 +0,0 @@ -package sqlancer.arangodb; - -import java.util.ArrayList; -import java.util.List; - -import com.arangodb.ArangoDB; -import com.arangodb.ArangoDatabase; -import com.google.auto.service.AutoService; - -import sqlancer.AbstractAction; -import sqlancer.DatabaseProvider; -import sqlancer.ExecutionTimer; -import sqlancer.GlobalState; -import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; -import sqlancer.Randomly; -import sqlancer.StatementExecutor; -import sqlancer.arangodb.gen.ArangoDBCreateIndexGenerator; -import sqlancer.arangodb.gen.ArangoDBInsertGenerator; -import sqlancer.arangodb.gen.ArangoDBTableGenerator; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.query.Query; - -@AutoService(DatabaseProvider.class) -public class ArangoDBProvider - extends ProviderAdapter { - - public ArangoDBProvider() { - super(ArangoDBGlobalState.class, ArangoDBOptions.class); - } - - enum Action implements AbstractAction { - INSERT(ArangoDBInsertGenerator::getQuery), CREATE_INDEX(ArangoDBCreateIndexGenerator::getQuery); - - private final ArangoDBQueryProvider queryProvider; - - Action(ArangoDBQueryProvider queryProvider) { - this.queryProvider = queryProvider; - } - - @Override - public Query getQuery(ArangoDBGlobalState globalState) throws Exception { - return queryProvider.getQuery(globalState); - } - } - - private static int mapActions(ArangoDBGlobalState globalState, Action a) { - Randomly r = globalState.getRandomly(); - switch (a) { - case INSERT: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case CREATE_INDEX: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumberIndexes); - default: - throw new AssertionError(a); - } - } - - public static class ArangoDBGlobalState extends GlobalState { - - private final List schemaTables = new ArrayList<>(); - - public void addTable(ArangoDBSchema.ArangoDBTable table) { - schemaTables.add(table); - } - - @Override - protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { - boolean logExecutionTime = getOptions().logExecutionTime(); - if (success && getOptions().printSucceedingStatements()) { - System.out.println(q.getLogString()); - } - if (logExecutionTime) { - getLogger().writeCurrent("//" + timer.end().asString()); - } - if (q.couldAffectSchema()) { - updateSchema(); - } - } - - @Override - protected ArangoDBSchema readSchema() throws Exception { - return new ArangoDBSchema(schemaTables); - } - } - - @Override - protected void checkViewsAreValid(ArangoDBGlobalState globalState) { - - } - - @Override - public void generateDatabase(ArangoDBGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { - boolean success; - do { - ArangoDBQueryAdapter queryAdapter = new ArangoDBTableGenerator().getQuery(globalState); - success = globalState.executeStatement(queryAdapter); - } while (!success); - } - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - ArangoDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public ArangoDBConnection createDatabase(ArangoDBGlobalState globalState) throws Exception { - ArangoDB arangoDB = new ArangoDB.Builder().user(globalState.getOptions().getUserName()) - .password(globalState.getOptions().getPassword()).build(); - ArangoDatabase database = arangoDB.db(globalState.getDatabaseName()); - try { - database.drop(); - // When the database does not exist, an ArangoDB exception is thrown. Since we are not sure - // if this is the first time the database is used, the simplest is dropping it and ignoring - // the exception. - } catch (Exception ignored) { - - } - arangoDB.createDatabase(globalState.getDatabaseName()); - database = arangoDB.db(globalState.getDatabaseName()); - return new ArangoDBConnection(arangoDB, database); - } - - @Override - public String getDBMSName() { - return "arangodb"; - } - - @Override - public LoggableFactory getLoggableFactory() { - return new ArangoDBLoggableFactory(); - } -} diff --git a/src/sqlancer/arangodb/ArangoDBQueryAdapter.java b/src/sqlancer/arangodb/ArangoDBQueryAdapter.java deleted file mode 100644 index 34cdb3709..000000000 --- a/src/sqlancer/arangodb/ArangoDBQueryAdapter.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.arangodb; - -import sqlancer.common.query.Query; - -public abstract class ArangoDBQueryAdapter extends Query { - @Override - public String getQueryString() { - // Should not be called as it is used only in SQL dependent classes - throw new UnsupportedOperationException(); - } - - @Override - public String getUnterminatedQueryString() { - throw new UnsupportedOperationException(); - } -} diff --git a/src/sqlancer/arangodb/ArangoDBQueryProvider.java b/src/sqlancer/arangodb/ArangoDBQueryProvider.java deleted file mode 100644 index 94a4ffda3..000000000 --- a/src/sqlancer/arangodb/ArangoDBQueryProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.arangodb; - -@FunctionalInterface -public interface ArangoDBQueryProvider { - ArangoDBQueryAdapter getQuery(S globalState) throws Exception; -} diff --git a/src/sqlancer/arangodb/ArangoDBSchema.java b/src/sqlancer/arangodb/ArangoDBSchema.java deleted file mode 100644 index 35e251b8b..000000000 --- a/src/sqlancer/arangodb/ArangoDBSchema.java +++ /dev/null @@ -1,70 +0,0 @@ -package sqlancer.arangodb; - -import java.util.Collections; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.common.schema.AbstractTables; -import sqlancer.common.schema.TableIndex; - -public class ArangoDBSchema extends AbstractSchema { - - public enum ArangoDBDataType { - INTEGER, DOUBLE, STRING, BOOLEAN; - - public static ArangoDBDataType getRandom() { - return Randomly.fromOptions(values()); - } - } - - public static class ArangoDBColumn extends AbstractTableColumn { - - private final boolean isId; - private final boolean isNullable; - - public ArangoDBColumn(String name, ArangoDBDataType type, boolean isId, boolean isNullable) { - super(name, null, type); - this.isId = isId; - this.isNullable = isNullable; - } - - public boolean isId() { - return isId; - } - - public boolean isNullable() { - return isNullable; - } - } - - public ArangoDBSchema(List databaseTables) { - super(databaseTables); - } - - public static class ArangoDBTables extends AbstractTables { - - public ArangoDBTables(List tables) { - super(tables); - } - } - - public static class ArangoDBTable - extends AbstractTable { - - public ArangoDBTable(String name, List columns, boolean isView) { - super(name, columns, Collections.emptyList(), isView); - } - - @Override - public long getNrRows(ArangoDBProvider.ArangoDBGlobalState globalState) { - throw new UnsupportedOperationException(); - } - } - - public ArangoDBTables getRandomTableNonEmptyTables() { - return new ArangoDBTables(Randomly.nonEmptySubset(getDatabaseTables())); - } -} diff --git a/src/sqlancer/arangodb/ast/ArangoDBConstant.java b/src/sqlancer/arangodb/ast/ArangoDBConstant.java deleted file mode 100644 index 351dbd822..000000000 --- a/src/sqlancer/arangodb/ast/ArangoDBConstant.java +++ /dev/null @@ -1,108 +0,0 @@ -package sqlancer.arangodb.ast; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.common.ast.newast.Node; - -public abstract class ArangoDBConstant implements Node { - private ArangoDBConstant() { - - } - - public abstract void setValueInDocument(BaseDocument document, String key); - - public abstract Object getValue(); - - public static class ArangoDBIntegerConstant extends ArangoDBConstant { - - private final int value; - - public ArangoDBIntegerConstant(int value) { - this.value = value; - } - - @Override - public void setValueInDocument(BaseDocument document, String key) { - document.addAttribute(key, value); - } - - @Override - public Object getValue() { - return value; - } - } - - public static Node createIntegerConstant(int value) { - return new ArangoDBIntegerConstant(value); - } - - public static class ArangoDBStringConstant extends ArangoDBConstant { - private final String value; - - public ArangoDBStringConstant(String value) { - this.value = value; - } - - @Override - public void setValueInDocument(BaseDocument document, String key) { - document.addAttribute(key, value); - } - - @Override - public Object getValue() { - return "'" + value.replace("\\", "\\\\").replace("'", "\\'") + "'"; - } - } - - public static Node createStringConstant(String value) { - return new ArangoDBStringConstant(value); - } - - public static class ArangoDBBooleanConstant extends ArangoDBConstant { - private final boolean value; - - public ArangoDBBooleanConstant(boolean value) { - this.value = value; - } - - @Override - public void setValueInDocument(BaseDocument document, String key) { - document.addAttribute(key, value); - } - - @Override - public Object getValue() { - return value; - } - } - - public static Node createBooleanConstant(boolean value) { - return new ArangoDBBooleanConstant(value); - } - - public static class ArangoDBDoubleConstant extends ArangoDBConstant { - private final double value; - - public ArangoDBDoubleConstant(double value) { - if (Double.isInfinite(value) || Double.isNaN(value)) { - this.value = 0.0; - } else { - this.value = value; - } - } - - @Override - public void setValueInDocument(BaseDocument document, String key) { - document.addAttribute(key, value); - } - - @Override - public Object getValue() { - return value; - } - } - - public static Node createDoubleConstant(double value) { - return new ArangoDBDoubleConstant(value); - } -} diff --git a/src/sqlancer/arangodb/ast/ArangoDBExpression.java b/src/sqlancer/arangodb/ast/ArangoDBExpression.java deleted file mode 100644 index facbbfe9e..000000000 --- a/src/sqlancer/arangodb/ast/ArangoDBExpression.java +++ /dev/null @@ -1,4 +0,0 @@ -package sqlancer.arangodb.ast; - -public interface ArangoDBExpression { -} diff --git a/src/sqlancer/arangodb/ast/ArangoDBSelect.java b/src/sqlancer/arangodb/ast/ArangoDBSelect.java deleted file mode 100644 index 9fb91d553..000000000 --- a/src/sqlancer/arangodb/ast/ArangoDBSelect.java +++ /dev/null @@ -1,79 +0,0 @@ -package sqlancer.arangodb.ast; - -import java.util.List; - -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.common.ast.newast.Node; - -public class ArangoDBSelect implements Node { - private List fromColumns; - private List projectionColumns; - private boolean hasFilter; - private Node filterClause; - private boolean hasComputed; - private List> computedClause; - - public List getFromColumns() { - if (fromColumns == null || fromColumns.isEmpty()) { - throw new IllegalStateException(); - } - return fromColumns; - } - - public void setFromColumns(List fromColumns) { - if (fromColumns == null || fromColumns.isEmpty()) { - throw new IllegalStateException(); - } - this.fromColumns = fromColumns; - } - - public List getProjectionColumns() { - if (projectionColumns == null) { - throw new IllegalStateException(); - } - return projectionColumns; - } - - public void setProjectionColumns(List projectionColumns) { - if (projectionColumns == null) { - throw new IllegalStateException(); - } - this.projectionColumns = projectionColumns; - } - - public void setFilterClause(Node filterClause) { - if (filterClause == null) { - hasFilter = false; - this.filterClause = null; - return; - } - hasFilter = true; - this.filterClause = filterClause; - } - - public Node getFilterClause() { - return filterClause; - } - - public boolean hasFilter() { - return hasFilter; - } - - public void setComputedClause(List> computedColumns) { - if (computedColumns == null || computedColumns.isEmpty()) { - hasComputed = false; - this.computedClause = null; - return; - } - hasComputed = true; - this.computedClause = computedColumns; - } - - public List> getComputedClause() { - return computedClause; - } - - public boolean hasComputed() { - return hasComputed; - } -} diff --git a/src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java b/src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java deleted file mode 100644 index eabd25578..000000000 --- a/src/sqlancer/arangodb/ast/ArangoDBUnsupportedPredicate.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.arangodb.ast; - -import sqlancer.common.ast.newast.Node; - -public class ArangoDBUnsupportedPredicate implements Node { -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java deleted file mode 100644 index 01e2e557a..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBComputedExpressionGenerator.java +++ /dev/null @@ -1,85 +0,0 @@ -package sqlancer.arangodb.gen; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBConstant; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; - -public class ArangoDBComputedExpressionGenerator - extends UntypedExpressionGenerator, ArangoDBSchema.ArangoDBColumn> { - private final ArangoDBProvider.ArangoDBGlobalState globalState; - - public ArangoDBComputedExpressionGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { - this.globalState = globalState; - } - - @Override - public Node generateConstant() { - ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); - switch (dataType) { - case INTEGER: - return ArangoDBConstant.createIntegerConstant((int) globalState.getRandomly().getInteger()); - case BOOLEAN: - return ArangoDBConstant.createBooleanConstant(Randomly.getBoolean()); - case DOUBLE: - return ArangoDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - case STRING: - return ArangoDBConstant.createStringConstant(globalState.getRandomly().getString()); - default: - throw new AssertionError(dataType); - } - } - - public enum ComputedFunction { - ADD(2, "+"), MINUS(2, "-"), MULTIPLY(2, "*"), DIVISION(2, "/"), MODULUS(2, "%"); - - private final int nrArgs; - private final String operatorName; - - ComputedFunction(int nrArgs, String operatorName) { - this.nrArgs = nrArgs; - this.operatorName = operatorName; - } - - public static ComputedFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - - public String getOperatorName() { - return operatorName; - } - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - ComputedFunction function = ComputedFunction.getRandom(); - return new NewFunctionNode<>(generateExpressions(function.getNrArgs(), depth + 1), function); - } - - @Override - protected Node generateColumn() { - return new ColumnReferenceNode<>(Randomly.fromList(columns)); - } - - @Override - public Node negatePredicate(Node predicate) { - throw new UnsupportedOperationException(); - } - - @Override - public Node isNull(Node expr) { - throw new UnsupportedOperationException(); - } -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java deleted file mode 100644 index 6a1b872da..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBCreateIndexGenerator.java +++ /dev/null @@ -1,18 +0,0 @@ -package sqlancer.arangodb.gen; - -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.query.ArangoDBCreateIndexQuery; - -public final class ArangoDBCreateIndexGenerator { - private ArangoDBCreateIndexGenerator() { - - } - - public static ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { - ArangoDBSchema.ArangoDBTable randomTable = globalState.getSchema().getRandomTable(); - ArangoDBSchema.ArangoDBColumn column = randomTable.getRandomColumn(); - return new ArangoDBCreateIndexQuery(column); - } -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java deleted file mode 100644 index 1a2fc4b5e..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBFilterExpressionGenerator.java +++ /dev/null @@ -1,153 +0,0 @@ -package sqlancer.arangodb.gen; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBConstant; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.arangodb.ast.ArangoDBUnsupportedPredicate; -import sqlancer.common.ast.BinaryOperatorNode; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; - -public class ArangoDBFilterExpressionGenerator - extends UntypedExpressionGenerator, ArangoDBSchema.ArangoDBColumn> { - - private final ArangoDBProvider.ArangoDBGlobalState globalState; - private int numberOfComputedVariables; - - private enum Expression { - BINARY_LOGICAL, UNARY_PREFIX, BINARY_COMPARISON - } - - public ArangoDBFilterExpressionGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { - this.globalState = globalState; - } - - public void setNumberOfComputedVariables(int numberOfComputedVariables) { - this.numberOfComputedVariables = numberOfComputedVariables; - } - - @Override - public Node generateConstant() { - ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); - switch (dataType) { - case INTEGER: - return ArangoDBConstant.createIntegerConstant((int) globalState.getRandomly().getInteger()); - case BOOLEAN: - return ArangoDBConstant.createBooleanConstant(Randomly.getBoolean()); - case DOUBLE: - return ArangoDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - case STRING: - return ArangoDBConstant.createStringConstant(globalState.getRandomly().getString()); - default: - throw new AssertionError(dataType); - } - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - Expression expression = Randomly.fromList(possibleOptions); - switch (expression) { - case BINARY_COMPARISON: - BinaryOperatorNode.Operator op = ArangoDBBinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), - ArangoDBUnaryPrefixOperator.getRandom()); - case BINARY_LOGICAL: - op = ArangoDBBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - default: - throw new AssertionError(expression); - } - } - - @Override - protected Node generateColumn() { - ArangoDBSchema.ArangoDBTable dummy = new ArangoDBSchema.ArangoDBTable("", new ArrayList<>(), false); - if (Randomly.getBoolean() || numberOfComputedVariables == 0) { - ArangoDBSchema.ArangoDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode<>(column); - } else { - int maxNumber = globalState.getRandomly().getInteger(0, numberOfComputedVariables); - ArangoDBSchema.ArangoDBColumn column = new ArangoDBSchema.ArangoDBColumn("c" + maxNumber, - ArangoDBSchema.ArangoDBDataType.INTEGER, false, false); - column.setTable(dummy); - return new ColumnReferenceNode<>(column); - } - } - - @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, ArangoDBUnaryPrefixOperator.NOT); - } - - @Override - public Node isNull(Node expr) { - return new ArangoDBUnsupportedPredicate<>(); - } - - public enum ArangoDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { - EQUALS("=="), NOT_EQUALS("!="), LESS_THAN("<"), LESS_OR_EQUAL("<="), GREATER_THAN(">"), GREATER_OR_EQUAL(">="); - - private final String representation; - - ArangoDBBinaryComparisonOperator(String representation) { - this.representation = representation; - } - - @Override - public String getTextRepresentation() { - return representation; - } - - public static ArangoDBBinaryComparisonOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum ArangoDBUnaryPrefixOperator implements BinaryOperatorNode.Operator { - NOT("!"); - - private final String representation; - - ArangoDBUnaryPrefixOperator(String representation) { - this.representation = representation; - } - - @Override - public String getTextRepresentation() { - return representation; - } - - public static ArangoDBUnaryPrefixOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum ArangoDBBinaryLogicalOperator implements BinaryOperatorNode.Operator { - AND, OR; - - @Override - public String getTextRepresentation() { - return toString(); - } - - public static BinaryOperatorNode.Operator getRandom() { - return Randomly.fromOptions(values()); - } - } - -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java deleted file mode 100644 index 3cfceeed4..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBInsertGenerator.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.arangodb.gen; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.query.ArangoDBConstantGenerator; -import sqlancer.arangodb.query.ArangoDBInsertQuery; - -public final class ArangoDBInsertGenerator { - - private final ArangoDBProvider.ArangoDBGlobalState globalState; - - private ArangoDBInsertGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { - this.globalState = globalState; - } - - public static ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { - return new ArangoDBInsertGenerator(globalState).generate(); - } - - private ArangoDBQueryAdapter generate() { - BaseDocument result = new BaseDocument(); - ArangoDBSchema.ArangoDBTable table = globalState.getSchema().getRandomTable(); - ArangoDBConstantGenerator constantGenerator = new ArangoDBConstantGenerator(globalState); - - for (int i = 0; i < table.getColumns().size(); i++) { - if (!globalState.getDbmsSpecificOptions().testRandomTypeInserts) { - constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), - table.getColumns().get(i).getType()); - } else { - constantGenerator.addRandomConstant(result, table.getColumns().get(i).getName()); - } - } - - return new ArangoDBInsertQuery(table, result); - } -} diff --git a/src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java b/src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java deleted file mode 100644 index 1236c3ce4..000000000 --- a/src/sqlancer/arangodb/gen/ArangoDBTableGenerator.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.arangodb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.query.ArangoDBCreateTableQuery; - -public class ArangoDBTableGenerator { - - private ArangoDBSchema.ArangoDBTable table; - private final List columnsToBeAdded = new ArrayList<>(); - - public ArangoDBQueryAdapter getQuery(ArangoDBProvider.ArangoDBGlobalState globalState) { - String tableName = globalState.getSchema().getFreeTableName(); - ArangoDBCreateTableQuery createTableQuery = new ArangoDBCreateTableQuery(tableName); - table = new ArangoDBSchema.ArangoDBTable(tableName, columnsToBeAdded, false); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - String columnName = String.format("c%d", i); - createColumn(columnName); - } - globalState.addTable(table); - return createTableQuery; - } - - private ArangoDBSchema.ArangoDBDataType createColumn(String columnName) { - ArangoDBSchema.ArangoDBDataType dataType = ArangoDBSchema.ArangoDBDataType.getRandom(); - ArangoDBSchema.ArangoDBColumn newColumn = new ArangoDBSchema.ArangoDBColumn(columnName, dataType, false, false); - newColumn.setTable(table); - columnsToBeAdded.add(newColumn); - return dataType; - } - - public String getTableName() { - return table.getName(); - } - - public ArangoDBSchema.ArangoDBTable getGeneratedTable() { - return table; - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java b/src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java deleted file mode 100644 index 406e8adca..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBConstantGenerator.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.arangodb.query; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBConstant; - -public class ArangoDBConstantGenerator { - private final ArangoDBProvider.ArangoDBGlobalState globalState; - - public ArangoDBConstantGenerator(ArangoDBProvider.ArangoDBGlobalState globalState) { - this.globalState = globalState; - } - - public void addRandomConstant(BaseDocument document, String key) { - ArangoDBSchema.ArangoDBDataType type = ArangoDBSchema.ArangoDBDataType.getRandom(); - addRandomConstantWithType(document, key, type); - } - - public void addRandomConstantWithType(BaseDocument document, String key, ArangoDBSchema.ArangoDBDataType dataType) { - ArangoDBConstant constant; - switch (dataType) { - case STRING: - constant = new ArangoDBConstant.ArangoDBStringConstant(globalState.getRandomly().getString()); - constant.setValueInDocument(document, key); - return; - case DOUBLE: - constant = new ArangoDBConstant.ArangoDBDoubleConstant(globalState.getRandomly().getDouble()); - constant.setValueInDocument(document, key); - return; - case BOOLEAN: - constant = new ArangoDBConstant.ArangoDBBooleanConstant(Randomly.getBoolean()); - constant.setValueInDocument(document, key); - return; - case INTEGER: - constant = new ArangoDBConstant.ArangoDBIntegerConstant((int) globalState.getRandomly().getInteger()); - constant.setValueInDocument(document, key); - return; - default: - throw new AssertionError(dataType); - } - - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java b/src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java deleted file mode 100644 index 6c2cc1b75..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBCreateIndexQuery.java +++ /dev/null @@ -1,54 +0,0 @@ -package sqlancer.arangodb.query; - -import java.util.Collections; - -import com.arangodb.ArangoCollection; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.arangodb.ArangoDBConnection; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.common.query.ExpectedErrors; - -public class ArangoDBCreateIndexQuery extends ArangoDBQueryAdapter { - - private final ArangoDBSchema.ArangoDBColumn column; - - public ArangoDBCreateIndexQuery(ArangoDBSchema.ArangoDBColumn column) { - this.column = column; - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - ArangoCollection collection = globalState.getConnection().getDatabase() - .collection(column.getTable().getName()); - collection.ensureHashIndex(Collections.singletonList(column.getName()), null); - Main.nrSuccessfulActions.addAndGet(1); - return true; - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - throw e; - } - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("db.").append(column.getTable().getName()) - .append(".ensureIndex({type: \"hash\", fields: [ \"").append(column.getName()).append("\" ]});"); - return stringBuilder.toString(); - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java b/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java deleted file mode 100644 index 00b3276d0..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBCreateTableQuery.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.arangodb.query; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.arangodb.ArangoDBConnection; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.common.query.ExpectedErrors; - -public class ArangoDBCreateTableQuery extends ArangoDBQueryAdapter { - - private final String tableName; - - public ArangoDBCreateTableQuery(String tableName) { - this.tableName = tableName; - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - globalState.getConnection().getDatabase().createCollection(tableName); - Main.nrSuccessfulActions.addAndGet(1); - return true; - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - throw e; - } - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - return "db._create(\"" + tableName + "\")"; - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java b/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java deleted file mode 100644 index 9a3612062..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBInsertQuery.java +++ /dev/null @@ -1,66 +0,0 @@ -package sqlancer.arangodb.query; - -import java.util.Map; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.arangodb.ArangoDBConnection; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.common.query.ExpectedErrors; - -public class ArangoDBInsertQuery extends ArangoDBQueryAdapter { - - private final ArangoDBSchema.ArangoDBTable table; - private final BaseDocument documentToBeInserted; - - public ArangoDBInsertQuery(ArangoDBSchema.ArangoDBTable table, BaseDocument documentToBeInserted) { - this.table = table; - this.documentToBeInserted = documentToBeInserted; - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - globalState.getConnection().getDatabase().collection(table.getName()).insertDocument(documentToBeInserted); - Main.nrSuccessfulActions.addAndGet(1); - return true; - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - throw e; - } - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("db._query(\"INSERT { "); - String filler = ""; - for (Map.Entry stringObjectEntry : documentToBeInserted.getProperties().entrySet()) { - stringBuilder.append(filler); - filler = ", "; - stringBuilder.append(stringObjectEntry.getKey()).append(": "); - Object value = stringObjectEntry.getValue(); - if (value instanceof String) { - stringBuilder.append("'").append(value).append("'"); - } else { - stringBuilder.append(value); - } - } - stringBuilder.append("} IN ").append(table.getName()).append("\")"); - return stringBuilder.toString(); - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java b/src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java deleted file mode 100644 index 835849b92..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBOptimizerRules.java +++ /dev/null @@ -1,57 +0,0 @@ -package sqlancer.arangodb.query; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; - -public class ArangoDBOptimizerRules { - - private final List allRules = new ArrayList<>(); - - public ArangoDBOptimizerRules() { - // SRC: - // https://www.arangodb.com/docs/stable/aql/execution-and-performance-optimizer.html#list-of-optimizer-rules - // Filtered out irrelevant ones - allRules.add("-fuse-filters"); - // allRules.add("-geo-index-optimizer"); - // allRules.add("-handle-arangosearch-views"); - // allRules.add("-inline-subqueries"); - allRules.add("-interchange-adjacent-enumerations"); - allRules.add("-late-document-materialization"); - // allRules.add("-late-document-materialization-arangosearch"); - allRules.add("-move-calculations-down"); - allRules.add("-move-calculations-up"); - allRules.add("-move-filters-into-enumerate"); - allRules.add("-move-filters-up"); - // allRules.add("-optimize-count"); - // allRules.add("-optimize-subqueries"); - // allRules.add("-optimize-traversals"); - // allRules.add("-patch-update-statements"); - allRules.add("-propagate-constant-attributes"); - allRules.add("-reduce-extraction-to-projection"); - // allRules.add("-remove-collect-variables"); - // allRules.add("-remove-data-modification-out-variables"); - allRules.add("-remove-filter-covered-by-index"); - // allRules.add("-remove-filter-covered-by-traversal"); - allRules.add("-remove-redundant-calculations"); - allRules.add("-remove-redundant-or"); - // allRules.add("-remove-redundant-path-var"); - // allRules.add("-remove-redundant-sorts"); - // allRules.add("-remove-sort-rand"); - allRules.add("-remove-unnecessary-calculations"); - allRules.add("-remove-unnecessary-filters"); - // allRules.add("-replace-function-with-index"); - allRules.add("-replace-or-with-in"); - allRules.add("-simplify-conditions"); - // allRules.add("-sort-in-values"); - // allRules.add("-sort-limit"); - // allRules.add("-splice-subqueries"); - // allRules.add("-use-index-for-sort"); - allRules.add("-use-indexes"); - } - - public List getRandomRules() { - return Randomly.subset(allRules); - } -} diff --git a/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java b/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java deleted file mode 100644 index 4725e4178..000000000 --- a/src/sqlancer/arangodb/query/ArangoDBSelectQuery.java +++ /dev/null @@ -1,88 +0,0 @@ -package sqlancer.arangodb.query; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import com.arangodb.ArangoCursor; -import com.arangodb.entity.BaseDocument; -import com.arangodb.model.AqlQueryOptions; - -import sqlancer.GlobalState; -import sqlancer.arangodb.ArangoDBConnection; -import sqlancer.arangodb.ArangoDBQueryAdapter; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLancerResultSet; - -public class ArangoDBSelectQuery extends ArangoDBQueryAdapter { - - private final String query; - - private List optimizerRules; - - private List resultSet; - - public ArangoDBSelectQuery(String query) { - this.query = query; - optimizerRules = new ArrayList<>(); - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - throw new UnsupportedOperationException(); - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - if (optimizerRules.isEmpty()) { - return "db._query(\"" + query + "\")"; - } else { - String rules = optimizerRules.stream().map(Object::toString).collect(Collectors.joining("\",\"")); - return "db._query(\"" + query + "\", null, { optimizer: { rules: [\"" + rules + "\"] } } )"; - } - } - - @Override - public > SQLancerResultSet executeAndGet(G globalState, - String... fills) throws Exception { - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(this.getLogString()); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - ArangoCursor cursor; - if (optimizerRules.isEmpty()) { - cursor = globalState.getConnection().getDatabase().query(query, BaseDocument.class); - } else { - AqlQueryOptions options = new AqlQueryOptions(); - cursor = globalState.getConnection().getDatabase().query(query, options.rules(optimizerRules), - BaseDocument.class); - } - resultSet = cursor.asListRemaining(); - return null; - } - - public List getResultSet() { - return resultSet; - } - - public void excludeRandomOptRules() { - optimizerRules = new ArangoDBOptimizerRules().getRandomRules(); - } -} diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java deleted file mode 100644 index 859ffc74e..000000000 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningBase.java +++ /dev/null @@ -1,67 +0,0 @@ -package sqlancer.arangodb.test; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.arangodb.ast.ArangoDBSelect; -import sqlancer.arangodb.gen.ArangoDBComputedExpressionGenerator; -import sqlancer.arangodb.gen.ArangoDBFilterExpressionGenerator; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; - -public class ArangoDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, ArangoDBProvider.ArangoDBGlobalState> - implements TestOracle { - - protected ArangoDBSchema schema; - protected List targetColumns; - protected ArangoDBFilterExpressionGenerator expressionGenerator; - protected ArangoDBSelect select; - protected int numberComputedColumns; - - protected ArangoDBQueryPartitioningBase(ArangoDBProvider.ArangoDBGlobalState state) { - super(state); - } - - @Override - protected ExpressionGenerator> getGen() { - return expressionGenerator; - } - - @Override - public void check() throws Exception { - numberComputedColumns = state.getRandomly().getInteger(0, 4); - schema = state.getSchema(); - generateTargetColumns(); - expressionGenerator = new ArangoDBFilterExpressionGenerator(state).setColumns(targetColumns); - expressionGenerator.setNumberOfComputedVariables(numberComputedColumns); - initializeTernaryPredicateVariants(); - select = new ArangoDBSelect<>(); - select.setFromColumns(targetColumns); - select.setProjectionColumns(Randomly.nonEmptySubset(targetColumns)); - generateComputedClause(); - } - - private void generateComputedClause() { - List> computedColumns = new ArrayList<>(); - ArangoDBComputedExpressionGenerator generator = new ArangoDBComputedExpressionGenerator(state); - generator.setColumns(targetColumns); - for (int i = 0; i < numberComputedColumns; i++) { - computedColumns.add(generator.generateExpression()); - } - select.setComputedClause(computedColumns); - } - - private void generateTargetColumns() { - ArangoDBSchema.ArangoDBTables targetTables; - targetTables = schema.getRandomTableNonEmptyTables(); - List allColumns = targetTables.getColumns(); - targetColumns = Randomly.nonEmptySubset(allColumns); - } -} diff --git a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java b/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java deleted file mode 100644 index 6ad19fabf..000000000 --- a/src/sqlancer/arangodb/test/ArangoDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.arangodb.test; - -import static sqlancer.arangodb.ArangoDBComparatorHelper.assumeResultSetsAreEqual; -import static sqlancer.arangodb.ArangoDBComparatorHelper.getResultSetAsDocumentList; - -import java.util.List; - -import com.arangodb.entity.BaseDocument; - -import sqlancer.arangodb.ArangoDBProvider; -import sqlancer.arangodb.query.ArangoDBSelectQuery; -import sqlancer.arangodb.visitor.ArangoDBVisitor; - -public class ArangoDBQueryPartitioningWhereTester extends ArangoDBQueryPartitioningBase { - public ArangoDBQueryPartitioningWhereTester(ArangoDBProvider.ArangoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - select.setFilterClause(null); - - ArangoDBSelectQuery query = ArangoDBVisitor.asSelectQuery(select); - List firstResultSet = getResultSetAsDocumentList(query, state); - - select.setFilterClause(predicate); - query = ArangoDBVisitor.asSelectQuery(select); - List secondResultSet = getResultSetAsDocumentList(query, state); - - select.setFilterClause(negatedPredicate); - query = ArangoDBVisitor.asSelectQuery(select); - List thirdResultSet = getResultSetAsDocumentList(query, state); - - thirdResultSet.addAll(secondResultSet); - assumeResultSetsAreEqual(firstResultSet, thirdResultSet, query); - - if (state.getDbmsSpecificOptions().withOptimizerRuleTests) { - select.setFilterClause(predicate); - query = ArangoDBVisitor.asSelectQuery(select); - query.excludeRandomOptRules(); - List forthResultSet = getResultSetAsDocumentList(query, state); - assumeResultSetsAreEqual(secondResultSet, forthResultSet, query); - } - } -} diff --git a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java deleted file mode 100644 index f82995d5e..000000000 --- a/src/sqlancer/arangodb/visitor/ArangoDBToQueryVisitor.java +++ /dev/null @@ -1,134 +0,0 @@ -package sqlancer.arangodb.visitor; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import sqlancer.arangodb.ArangoDBSchema; -import sqlancer.arangodb.ast.ArangoDBConstant; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.arangodb.ast.ArangoDBSelect; -import sqlancer.arangodb.gen.ArangoDBComputedExpressionGenerator; -import sqlancer.arangodb.query.ArangoDBSelectQuery; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; - -public class ArangoDBToQueryVisitor extends ArangoDBVisitor { - - private final StringBuilder stringBuilder; - - public ArangoDBToQueryVisitor() { - stringBuilder = new StringBuilder(); - } - - @Override - protected void visit(ArangoDBSelect expression) { - generateFrom(expression); - generateComputed(expression); - generateFilter(expression); - generateProject(expression); - } - - private void generateFilter(ArangoDBSelect expression) { - if (expression.hasFilter()) { - stringBuilder.append("FILTER "); - visit(expression.getFilterClause()); - stringBuilder.append(" "); - } - } - - private void generateComputed(ArangoDBSelect expression) { - if (expression.hasComputed()) { - List> computedClause = expression.getComputedClause(); - int computedNumber = 0; - for (Node computedExpression : computedClause) { - stringBuilder.append("LET c").append(computedNumber).append(" = "); - visit(computedExpression); - stringBuilder.append(" "); - computedNumber++; - } - } - } - - @Override - protected void visit(ColumnReferenceNode expression) { - if (expression.getColumn().getTable().getName().equals("")) { - stringBuilder.append(expression.getColumn().getName()); - } else { - stringBuilder.append("r").append(expression.getColumn().getTable().getName()).append(".") - .append(expression.getColumn().getName()); - } - } - - @Override - protected void visit(ArangoDBConstant expression) { - stringBuilder.append(expression.getValue()); - } - - @Override - protected void visit(NewBinaryOperatorNode expression) { - stringBuilder.append("("); - visit(expression.getLeft()); - stringBuilder.append(" ").append(expression.getOperatorRepresentation()).append(" "); - visit(expression.getRight()); - stringBuilder.append(")"); - } - - @Override - protected void visit(NewUnaryPrefixOperatorNode expression) { - stringBuilder.append(expression.getOperatorRepresentation()).append("("); - visit(expression.getExpr()); - stringBuilder.append(")"); - } - - @Override - protected void visit(NewFunctionNode expression) { - if (!(expression.getFunc() instanceof ArangoDBComputedExpressionGenerator.ComputedFunction)) { - throw new UnsupportedOperationException(); - } - ArangoDBComputedExpressionGenerator.ComputedFunction function = (ArangoDBComputedExpressionGenerator.ComputedFunction) expression - .getFunc(); - // TODO: Support functions with a different number of arguments. - if (function.getNrArgs() != 2) { - throw new UnsupportedOperationException(); - } - stringBuilder.append("("); - visit(expression.getArgs().get(0)); - stringBuilder.append(" ").append(function.getOperatorName()).append(" "); - visit(expression.getArgs().get(1)); - stringBuilder.append(")"); - } - - private void generateFrom(ArangoDBSelect expression) { - List forColumns = expression.getFromColumns(); - Set tables = new HashSet<>(); - for (ArangoDBSchema.ArangoDBColumn column : forColumns) { - tables.add(column.getTable()); - } - - for (ArangoDBSchema.ArangoDBTable table : tables) { - stringBuilder.append("FOR r").append(table.getName()).append(" IN ").append(table.getName()).append(" "); - } - } - - private void generateProject(ArangoDBSelect expression) { - List projectColumns = expression.getProjectionColumns(); - stringBuilder.append("RETURN {"); - String filler = ""; - for (ArangoDBSchema.ArangoDBColumn column : projectColumns) { - stringBuilder.append(filler); - filler = ", "; - stringBuilder.append(column.getTable().getName()).append("_").append(column.getName()).append(": r") - .append(column.getTable().getName()).append(".").append(column.getName()); - } - stringBuilder.append("}"); - } - - public ArangoDBSelectQuery getQuery() { - return new ArangoDBSelectQuery(stringBuilder.toString()); - } - -} diff --git a/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java b/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java deleted file mode 100644 index f1db84cf5..000000000 --- a/src/sqlancer/arangodb/visitor/ArangoDBVisitor.java +++ /dev/null @@ -1,51 +0,0 @@ -package sqlancer.arangodb.visitor; - -import sqlancer.arangodb.ast.ArangoDBConstant; -import sqlancer.arangodb.ast.ArangoDBExpression; -import sqlancer.arangodb.ast.ArangoDBSelect; -import sqlancer.arangodb.query.ArangoDBSelectQuery; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; - -public abstract class ArangoDBVisitor { - - protected abstract void visit(ArangoDBSelect expression); - - protected abstract void visit(ColumnReferenceNode expression); - - protected abstract void visit(ArangoDBConstant expression); - - protected abstract void visit(NewBinaryOperatorNode expression); - - protected abstract void visit(NewUnaryPrefixOperatorNode expression); - - protected abstract void visit(NewFunctionNode expression); - - @SuppressWarnings("unchecked") - public void visit(Node expressionNode) { - if (expressionNode instanceof ArangoDBSelect) { - visit((ArangoDBSelect) expressionNode); - } else if (expressionNode instanceof ColumnReferenceNode) { - visit((ColumnReferenceNode) expressionNode); - } else if (expressionNode instanceof ArangoDBConstant) { - visit((ArangoDBConstant) expressionNode); - } else if (expressionNode instanceof NewBinaryOperatorNode) { - visit((NewBinaryOperatorNode) expressionNode); - } else if (expressionNode instanceof NewUnaryPrefixOperatorNode) { - visit((NewUnaryPrefixOperatorNode) expressionNode); - } else if (expressionNode instanceof NewFunctionNode) { - visit((NewFunctionNode) expressionNode); - } else { - throw new AssertionError(expressionNode); - } - } - - public static ArangoDBSelectQuery asSelectQuery(Node expressionNode) { - ArangoDBToQueryVisitor visitor = new ArangoDBToQueryVisitor(); - visitor.visit(expressionNode); - return visitor.getQuery(); - } -} diff --git a/src/sqlancer/cosmos/CosmosProvider.java b/src/sqlancer/cosmos/CosmosProvider.java deleted file mode 100644 index 424625b4f..000000000 --- a/src/sqlancer/cosmos/CosmosProvider.java +++ /dev/null @@ -1,77 +0,0 @@ -package sqlancer.cosmos; - -import com.google.auto.service.AutoService; -import com.mongodb.ConnectionString; -import com.mongodb.MongoClientSettings; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoDatabase; - -import sqlancer.DatabaseProvider; -import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; -import sqlancer.Randomly; -import sqlancer.StatementExecutor; -import sqlancer.common.log.LoggableFactory; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBLoggableFactory; -import sqlancer.mongodb.MongoDBOptions; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.gen.MongoDBTableGenerator; - -@AutoService(DatabaseProvider.class) -public class CosmosProvider extends - ProviderAdapter { - - public CosmosProvider() { - super(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState.class, MongoDBOptions.class); - } - - @Override - public void generateDatabase(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { - boolean success; - do { - MongoDBQueryAdapter query = new MongoDBTableGenerator(globalState).getQuery(globalState); - success = globalState.executeStatement(query); - } while (!success); - } - StatementExecutor se = new StatementExecutor<>( - globalState, sqlancer.mongodb.MongoDBProvider.Action.values(), - sqlancer.mongodb.MongoDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public MongoDBConnection createDatabase(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState globalState) - throws Exception { - String connectionString = ""; - if (connectionString.equals("")) { - throw new AssertionError("Please set connection string for cosmos database, located in CosmosProvider"); - } - MongoClientSettings settings = MongoClientSettings.builder() - .applyConnectionString(new ConnectionString(connectionString)).build(); - MongoClient mongoClient = MongoClients.create(settings); - MongoDatabase database = mongoClient.getDatabase(globalState.getDatabaseName()); - database.drop(); - return new MongoDBConnection(mongoClient, database); - } - - @Override - public String getDBMSName() { - return "cosmos"; - } - - @Override - public LoggableFactory getLoggableFactory() { - return new MongoDBLoggableFactory(); - } - - @Override - protected void checkViewsAreValid(sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState globalState) { - } -} diff --git a/src/sqlancer/mongodb/MongoDBComparatorHelper.java b/src/sqlancer/mongodb/MongoDBComparatorHelper.java deleted file mode 100644 index 49b692645..000000000 --- a/src/sqlancer/mongodb/MongoDBComparatorHelper.java +++ /dev/null @@ -1,97 +0,0 @@ -package sqlancer.mongodb; - -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.bson.Document; - -import sqlancer.IgnoreMeException; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.query.MongoDBSelectQuery; - -public final class MongoDBComparatorHelper { - - private MongoDBComparatorHelper() { - } - - public static List getResultSetAsDocumentList(MongoDBSelectQuery adapter, MongoDBGlobalState state) - throws Exception { - ExpectedErrors errors = adapter.getExpectedErrors(); - List result; - try { - adapter.executeAndGet(state); - Main.nrSuccessfulActions.addAndGet(1); - result = adapter.getResultSet(); - return result; - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw e; - } - Main.nrUnsuccessfulActions.addAndGet(1); - if (e.getMessage() == null) { - throw new AssertionError(adapter.getLogString(), e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(adapter.getLogString(), e); - } - } - - public static void assumeCountIsEqual(List resultSet, List secondResultSet, - MongoDBSelectQuery originalQuery) { - int originalSize = resultSet.size(); - if (secondResultSet.isEmpty()) { - if (originalSize == 0) { - return; - } else { - String assertMessage = String.format("The Count of the result set mismatches!\n %s", - originalQuery.getLogString()); - throw new AssertionError(assertMessage); - } - } - if (secondResultSet.size() != 1) { - throw new AssertionError( - String.format("Count query result bigger than one \n %s", originalQuery.getLogString())); - } - int withCount = (int) secondResultSet.get(0).get("count"); - if (originalSize != withCount) { - String assertMessage = String.format("The Count of the result set mismatches!\n %s", - originalQuery.getLogString()); - throw new AssertionError(assertMessage); - } - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - MongoDBSelectQuery originalQuery) { - if (resultSet.size() != secondResultSet.size()) { - String assertionMessage = String.format("The Size of the result sets mismatch (%d and %d)!\n%s", - resultSet.size(), secondResultSet.size(), originalQuery.getLogString()); - throw new AssertionError(assertionMessage); - } - - Set firstHashSet = new HashSet<>(resultSet); - Set secondHashSet = new HashSet<>(secondResultSet); - - if (!firstHashSet.equals(secondHashSet)) { - Set firstResultSetMisses = new HashSet<>(firstHashSet); - firstResultSetMisses.removeAll(secondHashSet); - Set secondResultSetMisses = new HashSet<>(secondHashSet); - secondResultSetMisses.removeAll(firstHashSet); - StringBuilder firstMisses = new StringBuilder(); - for (Document document : firstResultSetMisses) { - firstMisses.append(document.toJson()).append(" "); - } - StringBuilder secondMisses = new StringBuilder(); - for (Document document : secondResultSetMisses) { - secondMisses.append(document.toJson()).append(" "); - } - String assertMessage = String.format("The Content of the result sets mismatch!\n %s \n %s\n %s", - firstMisses.toString(), secondMisses.toString(), originalQuery.getLogString()); - throw new AssertionError(assertMessage); - } - } -} diff --git a/src/sqlancer/mongodb/MongoDBConnection.java b/src/sqlancer/mongodb/MongoDBConnection.java deleted file mode 100644 index 6971bd79c..000000000 --- a/src/sqlancer/mongodb/MongoDBConnection.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.mongodb; - -import org.bson.BsonDocument; -import org.bson.BsonString; - -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoDatabase; - -import sqlancer.SQLancerDBConnection; - -public class MongoDBConnection implements SQLancerDBConnection { - - private final MongoClient client; - private final MongoDatabase database; - - public MongoDBConnection(MongoClient client, MongoDatabase database) { - this.client = client; - this.database = database; - } - - @Override - public String getDatabaseVersion() throws Exception { - return client.getDatabase("dbname").runCommand(new BsonDocument("buildinfo", new BsonString(""))).get("version") - .toString(); - } - - @Override - public void close() throws Exception { - client.close(); - } - - public MongoDatabase getDatabase() { - return database; - } -} diff --git a/src/sqlancer/mongodb/MongoDBLoggableFactory.java b/src/sqlancer/mongodb/MongoDBLoggableFactory.java deleted file mode 100644 index b668301b3..000000000 --- a/src/sqlancer/mongodb/MongoDBLoggableFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -package sqlancer.mongodb; - -import java.util.Arrays; - -import sqlancer.common.log.Loggable; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.log.LoggedString; -import sqlancer.common.query.Query; - -public class MongoDBLoggableFactory extends LoggableFactory { - @Override - protected Loggable createLoggable(String input, String suffix) { - return new LoggedString(input + suffix); - } - - @Override - public Query getQueryForStateToReproduce(String queryString) { - throw new UnsupportedOperationException(); - } - - @Override - public Query commentOutQuery(Query query) { - throw new UnsupportedOperationException(); - } - - @Override - protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { - StringBuilder sb = new StringBuilder(); - sb.append("// Time: ").append(time).append("\n"); - sb.append("// Database: ").append(databaseName).append("\n"); - sb.append("// Database version: ").append(databaseVersion).append("\n"); - sb.append("// seed value: ").append(seedValue).append("\n"); - return new LoggedString(sb.toString()); - } - - @Override - public Loggable convertStacktraceToLoggable(Throwable throwable) { - return new LoggedString(Arrays.toString(throwable.getStackTrace()) + "\n" + throwable.getMessage()); - } -} diff --git a/src/sqlancer/mongodb/MongoDBOptions.java b/src/sqlancer/mongodb/MongoDBOptions.java deleted file mode 100644 index 78bd73ffc..000000000 --- a/src/sqlancer/mongodb/MongoDBOptions.java +++ /dev/null @@ -1,74 +0,0 @@ -package sqlancer.mongodb; - -import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.DOCUMENT_REMOVAL; -import static sqlancer.mongodb.MongoDBOptions.MongoDBOracleFactory.QUERY_PARTITIONING; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.test.MongoDBDocumentRemovalTester; -import sqlancer.mongodb.test.MongoDBQueryPartitioningWhereTester; - -@Parameters(commandDescription = "MongoDB (experimental)") -public class MongoDBOptions implements DBMSSpecificOptions { - - @Parameter(names = "--test-validation", description = "Enable/Disable validation of schema with Schema Validation", arity = 1) - public boolean testValidation = true; - - @Parameter(names = "--test-null-inserts", description = "Enables to test inserting with null values, validation has to be off", arity = 1) - public boolean testNullInserts; - - @Parameter(names = "--test-random-types", description = "Insert random types instead of schema types, validation has to be off", arity = 1) - public boolean testRandomTypes; - - @Parameter(names = "--max-number-indexes", description = "The maximum number of indexes used.", arity = 1) - public int maxNumberIndexes = 15; - - @Parameter(names = "--test-computed-values", description = "Enable adding computed values to query", arity = 1) - public boolean testComputedValues; - - @Parameter(names = "--test-with-regex", description = "Enable Regex Leaf Nodes", arity = 1) - public boolean testWithRegex; - - @Parameter(names = "--test-with-count", description = "Count the number of documents and check with count command", arity = 1) - public boolean testWithCount; - - @Parameter(names = "--null-safety", description = "", arity = 1) - public boolean nullSafety; - - @Parameter(names = "--oracle") - public List oracles = Arrays.asList(QUERY_PARTITIONING, DOCUMENT_REMOVAL); - - @Override - public List getTestOracleFactory() { - return oracles; - } - - public enum MongoDBOracleFactory implements OracleFactory { - QUERY_PARTITIONING { - @Override - public TestOracle create(MongoDBGlobalState globalState) throws Exception { - List> oracles = new ArrayList<>(); - oracles.add(new MongoDBQueryPartitioningWhereTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - DOCUMENT_REMOVAL { - @Override - public TestOracle create(MongoDBGlobalState globalState) throws Exception { - List> oracles = new ArrayList<>(); - oracles.add(new MongoDBDocumentRemovalTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - } - } -} diff --git a/src/sqlancer/mongodb/MongoDBProvider.java b/src/sqlancer/mongodb/MongoDBProvider.java deleted file mode 100644 index 09662a863..000000000 --- a/src/sqlancer/mongodb/MongoDBProvider.java +++ /dev/null @@ -1,128 +0,0 @@ -package sqlancer.mongodb; - -import java.util.ArrayList; -import java.util.List; - -import com.google.auto.service.AutoService; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoDatabase; - -import sqlancer.AbstractAction; -import sqlancer.DatabaseProvider; -import sqlancer.ExecutionTimer; -import sqlancer.GlobalState; -import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; -import sqlancer.Randomly; -import sqlancer.StatementExecutor; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.query.Query; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.gen.MongoDBIndexGenerator; -import sqlancer.mongodb.gen.MongoDBInsertGenerator; -import sqlancer.mongodb.gen.MongoDBTableGenerator; - -@AutoService(DatabaseProvider.class) -public class MongoDBProvider - extends ProviderAdapter { - - public MongoDBProvider() { - super(MongoDBGlobalState.class, MongoDBOptions.class); - } - - public enum Action implements AbstractAction { - INSERT(MongoDBInsertGenerator::getQuery), CREATE_INDEX(MongoDBIndexGenerator::getQuery); - - private final MongoDBQueryProvider queryProvider; - - Action(MongoDBQueryProvider queryProvider) { - this.queryProvider = queryProvider; - } - - @Override - public Query getQuery(MongoDBGlobalState globalState) throws Exception { - return queryProvider.getQuery(globalState); - } - } - - public static int mapActions(MongoDBGlobalState globalState, Action a) { - Randomly r = globalState.getRandomly(); - switch (a) { - case INSERT: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case CREATE_INDEX: - return r.getInteger(0, globalState.getDbmsSpecificOptions().maxNumberIndexes); - default: - throw new AssertionError(a); - } - } - - public static class MongoDBGlobalState extends GlobalState { - - private final List schemaTables = new ArrayList<>(); - - public void addTable(MongoDBTable table) { - schemaTables.add(table); - } - - @Override - protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { - boolean logExecutionTime = getOptions().logExecutionTime(); - if (success && getOptions().printSucceedingStatements()) { - System.out.println(q.getLogString()); - } - if (logExecutionTime) { - getLogger().writeCurrent("// " + timer.end().asString()); - } - if (q.couldAffectSchema()) { - updateSchema(); - } - } - - @Override - protected MongoDBSchema readSchema() throws Exception { - return new MongoDBSchema(schemaTables); - } - } - - @Override - public void generateDatabase(MongoDBGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(4, 5, 6); i++) { - boolean success; - do { - MongoDBQueryAdapter query = new MongoDBTableGenerator(globalState).getQuery(globalState); - success = globalState.executeStatement(query); - } while (!success); - } - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - MongoDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public MongoDBConnection createDatabase(MongoDBGlobalState globalState) throws Exception { - MongoClient mongoClient = MongoClients.create(); - MongoDatabase database = mongoClient.getDatabase(globalState.getDatabaseName()); - database.drop(); - return new MongoDBConnection(mongoClient, database); - } - - @Override - public String getDBMSName() { - return "mongodb"; - } - - @Override - public LoggableFactory getLoggableFactory() { - return new MongoDBLoggableFactory(); - } - - @Override - protected void checkViewsAreValid(MongoDBGlobalState globalState) { - } -} diff --git a/src/sqlancer/mongodb/MongoDBQueryAdapter.java b/src/sqlancer/mongodb/MongoDBQueryAdapter.java deleted file mode 100644 index e2add3242..000000000 --- a/src/sqlancer/mongodb/MongoDBQueryAdapter.java +++ /dev/null @@ -1,15 +0,0 @@ -package sqlancer.mongodb; - -import sqlancer.common.query.Query; - -public abstract class MongoDBQueryAdapter extends Query { - @Override - public String getQueryString() { - throw new UnsupportedOperationException(); - } - - @Override - public String getUnterminatedQueryString() { - throw new UnsupportedOperationException(); - } -} diff --git a/src/sqlancer/mongodb/MongoDBQueryProvider.java b/src/sqlancer/mongodb/MongoDBQueryProvider.java deleted file mode 100644 index 970c90cea..000000000 --- a/src/sqlancer/mongodb/MongoDBQueryProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.mongodb; - -@FunctionalInterface -public interface MongoDBQueryProvider { - MongoDBQueryAdapter getQuery(S globalState) throws Exception; -} diff --git a/src/sqlancer/mongodb/MongoDBSchema.java b/src/sqlancer/mongodb/MongoDBSchema.java deleted file mode 100644 index 5ae3cdd24..000000000 --- a/src/sqlancer/mongodb/MongoDBSchema.java +++ /dev/null @@ -1,97 +0,0 @@ -package sqlancer.mongodb; - -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.bson.BsonType; - -import com.mongodb.client.MongoDatabase; - -import sqlancer.Randomly; -import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.common.schema.AbstractTables; -import sqlancer.common.schema.TableIndex; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; - -public class MongoDBSchema extends AbstractSchema { - - public enum MongoDBDataType { - INTEGER(BsonType.INT32), STRING(BsonType.STRING), BOOLEAN(BsonType.BOOLEAN), DOUBLE(BsonType.DOUBLE), - DATE_TIME(BsonType.DATE_TIME), TIMESTAMP(BsonType.TIMESTAMP); - - private final BsonType bsonType; - - MongoDBDataType(BsonType type) { - this.bsonType = type; - } - - public BsonType getBsonType() { - return bsonType; - } - - public static MongoDBDataType getRandom(MongoDBGlobalState state) { - Set valueSet = new HashSet<>(Arrays.asList(values())); - if (state.getDbmsSpecificOptions().nullSafety) { - valueSet.remove(STRING); - } - MongoDBDataType[] configuredValues = new MongoDBDataType[valueSet.size()]; - return Randomly.fromOptions(valueSet.toArray(configuredValues)); - } - } - - public static class MongoDBColumn extends AbstractTableColumn { - - private final boolean isId; - private final boolean isNullable; - - public MongoDBColumn(String name, MongoDBDataType type, boolean isId, boolean isNullable) { - super(name, null, type); - this.isId = isId; - this.isNullable = isNullable; - } - - public boolean isId() { - return isId; - } - - public boolean isNullable() { - return isNullable; - } - - } - - public static class MongoDBTables extends AbstractTables { - - public MongoDBTables(List tables) { - super(tables); - } - } - - public MongoDBSchema(List databaseTables) { - super(databaseTables); - } - - public static class MongoDBTable extends AbstractTable { - public MongoDBTable(String name, List columns, boolean isView) { - super(name, columns, Collections.emptyList(), isView); - } - - @Override - public long getNrRows(MongoDBGlobalState globalState) { - throw new UnsupportedOperationException(); - } - } - - public static MongoDBSchema fromConnection(MongoDatabase connection, String databaseName) { - throw new UnsupportedOperationException(); - } - - public MongoDBTables getRandomTableNonEmptyTables() { - return new MongoDBTables(Randomly.nonEmptySubset(getDatabaseTables())); - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java b/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java deleted file mode 100644 index 21675250a..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBBinaryComparisonNode.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.mongodb.ast; - -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator; - -public class MongoDBBinaryComparisonNode extends NewBinaryOperatorNode { - public MongoDBBinaryComparisonNode(Node left, Node right, - MongoDBBinaryComparisonOperator op) { - super(left, right, op); - } - - public MongoDBBinaryComparisonOperator operator() { - return (MongoDBBinaryComparisonOperator) op; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java b/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java deleted file mode 100644 index efb8d8294..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBBinaryLogicalNode.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.mongodb.ast; - -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator; - -public class MongoDBBinaryLogicalNode extends NewBinaryOperatorNode { - public MongoDBBinaryLogicalNode(Node left, Node right, - MongoDBBinaryLogicalOperator op) { - super(left, right, op); - } - - public MongoDBBinaryLogicalOperator operator() { - return (MongoDBBinaryLogicalOperator) op; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBConstant.java b/src/sqlancer/mongodb/ast/MongoDBConstant.java deleted file mode 100644 index 86f783b48..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBConstant.java +++ /dev/null @@ -1,252 +0,0 @@ -package sqlancer.mongodb.ast; - -import java.io.Serializable; - -import org.bson.BsonDateTime; -import org.bson.BsonTimestamp; -import org.bson.Document; - -import sqlancer.common.ast.newast.Node; - -public abstract class MongoDBConstant implements Node { - private MongoDBConstant() { - } - - public abstract void setValueInDocument(Document document, String key); - - public abstract String getLogValue(); - - public abstract Object getValue(); - - public abstract Serializable getSerializedValue(); - - public static class MongoDBNullConstant extends MongoDBConstant { - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, null); - } - - @Override - public String getLogValue() { - return "null"; - } - - @Override - public Object getValue() { - return null; - } - - @Override - public Serializable getSerializedValue() { - return null; - } - } - - public static Node createNullConstant() { - return new MongoDBNullConstant(); - } - - public static class MongoDBIntegerConstant extends MongoDBConstant { - - private final int value; - - public MongoDBIntegerConstant(int value) { - this.value = value; - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return "NumberInt(" + value + ")"; - } - - @Override - public Integer getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value; - } - } - - public static Node createIntegerConstant(int value) { - return new MongoDBIntegerConstant(value); - } - - public static class MongoDBStringConstant extends MongoDBConstant { - - private final String value; - - public MongoDBStringConstant(String value) { - this.value = value; - } - - public String getStringValue() { - return value; - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return "\"" + value.replace("\\", "\\\\").replace("\"", "\\\"").replace("\n", "\\n") + "\""; - } - - @Override - public String getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value; - } - } - - public static Node createStringConstant(String value) { - return new MongoDBStringConstant(value); - } - - public static class MongoDBBooleanConstant extends MongoDBConstant { - - private final boolean value; - - public MongoDBBooleanConstant(boolean value) { - this.value = value; - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return String.valueOf(value); - } - - @Override - public Boolean getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value; - } - } - - public static Node createBooleanConstant(boolean value) { - return new MongoDBBooleanConstant(value); - } - - public static class MongoDBDoubleConstant extends MongoDBConstant { - - private final double value; - - public MongoDBDoubleConstant(double value) { - this.value = value; - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return String.valueOf(value); - } - - @Override - public Double getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value; - } - } - - public static Node createDoubleConstant(double value) { - return new MongoDBDoubleConstant(value); - } - - public static class MongoDBDateTimeConstant extends MongoDBConstant { - - private final BsonDateTime value; - - public MongoDBDateTimeConstant(long val) { - this.value = new BsonDateTime(val); - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return "new Date(" + value.getValue() + ")"; - } - - @Override - public BsonDateTime getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value.getValue(); - } - } - - public static Node createDateTimeConstant(long value) { - return new MongoDBDateTimeConstant(value); - } - - public static class MongoDBTimestampConstant extends MongoDBConstant { - - private final BsonTimestamp value; - - public MongoDBTimestampConstant(long value) { - this.value = new BsonTimestamp(value); - } - - @Override - public void setValueInDocument(Document document, String key) { - document.append(key, value); - } - - @Override - public String getLogValue() { - return "Timestamp(" + value.getValue() + ",1)"; - } - - @Override - public BsonTimestamp getValue() { - return value; - } - - @Override - public Serializable getSerializedValue() { - return value.getValue(); - } - } - - public static Node createTimestampConstant(long value) { - return new MongoDBTimestampConstant(value); - } - -} diff --git a/src/sqlancer/mongodb/ast/MongoDBExpression.java b/src/sqlancer/mongodb/ast/MongoDBExpression.java deleted file mode 100644 index 1235a1fbc..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBExpression.java +++ /dev/null @@ -1,4 +0,0 @@ -package sqlancer.mongodb.ast; - -public interface MongoDBExpression { -} diff --git a/src/sqlancer/mongodb/ast/MongoDBRegexNode.java b/src/sqlancer/mongodb/ast/MongoDBRegexNode.java deleted file mode 100644 index 76c608586..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBRegexNode.java +++ /dev/null @@ -1,24 +0,0 @@ -package sqlancer.mongodb.ast; - -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBRegexOperator.REGEX; - -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBRegexOperator; - -public class MongoDBRegexNode extends NewBinaryOperatorNode { - private final String options; - - public MongoDBRegexNode(Node left, Node right, String options) { - super(left, right, REGEX); - this.options = options; - } - - public String getOptions() { - return options; - } - - public MongoDBRegexOperator operator() { - return (MongoDBRegexOperator) op; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBSelect.java b/src/sqlancer/mongodb/ast/MongoDBSelect.java deleted file mode 100644 index 0fe91ba4a..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBSelect.java +++ /dev/null @@ -1,104 +0,0 @@ -package sqlancer.mongodb.ast; - -import java.util.List; - -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.test.MongoDBColumnTestReference; - -public class MongoDBSelect implements Node { - - private final String mainTableName; - private final MongoDBColumnTestReference joinColumn; - List projectionColumns; - List lookupList; - boolean hasFilter; - Node filterClause; - boolean hasComputed; - List> computedClauses; - private boolean withCountClause; - - public MongoDBSelect(String mainTableName, MongoDBColumnTestReference joinColumn) { - this.mainTableName = mainTableName; - this.joinColumn = joinColumn; - } - - public String getMainTableName() { - return mainTableName; - } - - public MongoDBColumnTestReference getJoinColumn() { - return joinColumn; - } - - public void setProjectionList(List fetchColumns) { - if (fetchColumns == null || fetchColumns.isEmpty()) { - throw new IllegalArgumentException(); - } - this.projectionColumns = fetchColumns; - } - - public List getProjectionList() { - if (projectionColumns == null) { - throw new IllegalStateException(); - } - return projectionColumns; - } - - public void setLookupList(List lookupList) { - if (lookupList == null || lookupList.isEmpty()) { - throw new IllegalArgumentException(); - } - this.lookupList = lookupList; - } - - public List getLookupList() { - if (lookupList == null) { - throw new IllegalStateException(); - } - return lookupList; - } - - public void setFilterClause(Node filterClause) { - if (filterClause == null) { - hasFilter = false; - this.filterClause = null; - return; - } - hasFilter = true; - this.filterClause = filterClause; - } - - public Node getFilterClause() { - return filterClause; - } - - public boolean hasFilter() { - return hasFilter; - } - - public void setComputedClause(List> computedClause) { - if (computedClause == null) { - hasComputed = false; - this.computedClauses = null; - return; - } - hasComputed = true; - this.computedClauses = computedClause; - } - - public List> getComputedClause() { - return computedClauses; - } - - public boolean hasComputed() { - return hasComputed; - } - - public boolean getWithCountClause() { - return withCountClause; - } - - public void setWithCountClause(boolean withCountClause) { - this.withCountClause = withCountClause; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java b/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java deleted file mode 100644 index a34fe27e5..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBUnaryLogicalOperatorNode.java +++ /dev/null @@ -1,16 +0,0 @@ -package sqlancer.mongodb.ast; - -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBUnaryLogicalOperator; - -public class MongoDBUnaryLogicalOperatorNode extends NewUnaryPrefixOperatorNode { - - public MongoDBUnaryLogicalOperatorNode(Node expr, MongoDBUnaryLogicalOperator op) { - super(expr, op); - } - - public MongoDBUnaryLogicalOperator operator() { - return (MongoDBUnaryLogicalOperator) op; - } -} diff --git a/src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java b/src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java deleted file mode 100644 index eae143e7d..000000000 --- a/src/sqlancer/mongodb/ast/MongoDBUnsupportedPredicate.java +++ /dev/null @@ -1,7 +0,0 @@ -package sqlancer.mongodb.ast; - -import sqlancer.common.ast.newast.Node; - -public class MongoDBUnsupportedPredicate implements Node { - -} diff --git a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java deleted file mode 100644 index 347d5bc97..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBComputedExpressionGenerator.java +++ /dev/null @@ -1,89 +0,0 @@ -package sqlancer.mongodb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBSchema; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.test.MongoDBColumnTestReference; - -public class MongoDBComputedExpressionGenerator - extends UntypedExpressionGenerator, MongoDBColumnTestReference> { - - private final MongoDBGlobalState globalState; - - @Override - public Node generateLeafNode() { - ComputedFunction function = ComputedFunction.getRandom(); - List> expressions = new ArrayList<>(); - for (int i = 0; i < function.getNrArgs(); i++) { - expressions.add(super.generateLeafNode()); - } - return new NewFunctionNode<>(expressions, function); - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - ComputedFunction func = ComputedFunction.getRandom(); - return new NewFunctionNode<>(generateExpressions(func.getNrArgs(), depth + 1), func); - } - - public MongoDBComputedExpressionGenerator(MongoDBGlobalState globalState) { - this.globalState = globalState; - } - - public enum ComputedFunction { - ADD(2, "$add"), MULTIPLY(2, "$multiply"), DIVIDE(2, "$divide"), POW(2, "$pow"), SQRT(1, "$sqrt"), - LOG(2, "$log"), AVG(2, "$avg"), EXP(1, "$exp"); - - private final int nrArgs; - private final String operatorName; - - ComputedFunction(int nrArgs, String operatorName) { - this.nrArgs = nrArgs; - this.operatorName = operatorName; - } - - public static ComputedFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - - public String getOperator() { - return operatorName; - } - } - - @Override - public Node generateConstant() { - MongoDBSchema.MongoDBDataType type = MongoDBSchema.MongoDBDataType.getRandom(globalState); - MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); - return generator.generateConstantWithType(type); - } - - @Override - protected Node generateColumn() { - return Randomly.fromList(columns); - } - - @Override - public Node negatePredicate(Node predicate) { - throw new UnsupportedOperationException(); - } - - @Override - public Node isNull(Node expr) { - throw new UnsupportedOperationException(); - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java b/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java deleted file mode 100644 index 2e6b15048..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBConstantGenerator.java +++ /dev/null @@ -1,86 +0,0 @@ -package sqlancer.mongodb.gen; - -import org.bson.Document; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBBooleanConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBDateTimeConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBDoubleConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBIntegerConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBNullConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBTimestampConstant; -import sqlancer.mongodb.ast.MongoDBExpression; - -public class MongoDBConstantGenerator { - private final MongoDBGlobalState globalState; - - public MongoDBConstantGenerator(MongoDBGlobalState globalState) { - this.globalState = globalState; - } - - public Node generateConstantWithType(MongoDBDataType option) { - switch (option) { - case DATE_TIME: - return MongoDBConstant.createDateTimeConstant(globalState.getRandomly().getInteger()); - case BOOLEAN: - return MongoDBConstant.createBooleanConstant(Randomly.getBoolean()); - case DOUBLE: - return MongoDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - case STRING: - return MongoDBConstant.createStringConstant(globalState.getRandomly().getString()); - case INTEGER: - return MongoDBConstant.createIntegerConstant((int) globalState.getRandomly().getInteger()); - case TIMESTAMP: - return MongoDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); - default: - throw new AssertionError(option); - } - } - - public void addRandomConstant(Document document, String key) { - MongoDBDataType type = MongoDBDataType.getRandom(globalState); - addRandomConstantWithType(document, key, type); - } - - public void addRandomConstantWithType(Document document, String key, MongoDBDataType option) { - MongoDBConstant constant; - if (globalState.getDbmsSpecificOptions().testNullInserts && Randomly.getBooleanWithSmallProbability()) { - constant = new MongoDBNullConstant(); - constant.setValueInDocument(document, key); - return; - } - switch (option) { - case DATE_TIME: - constant = new MongoDBDateTimeConstant(globalState.getRandomly().getInteger()); - constant.setValueInDocument(document, key); - return; - - case BOOLEAN: - constant = new MongoDBBooleanConstant(Randomly.getBoolean()); - constant.setValueInDocument(document, key); - return; - case DOUBLE: - constant = new MongoDBDoubleConstant(globalState.getRandomly().getDouble()); - constant.setValueInDocument(document, key); - return; - case STRING: - constant = new MongoDBConstant.MongoDBStringConstant(globalState.getRandomly().getString()); - constant.setValueInDocument(document, key); - return; - case INTEGER: - constant = new MongoDBIntegerConstant((int) globalState.getRandomly().getInteger()); - constant.setValueInDocument(document, key); - return; - case TIMESTAMP: - constant = new MongoDBTimestampConstant(globalState.getRandomly().getInteger()); - constant.setValueInDocument(document, key); - return; - default: - throw new AssertionError(option); - } - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java b/src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java deleted file mode 100644 index 8687fd45c..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBIndexGenerator.java +++ /dev/null @@ -1,25 +0,0 @@ -package sqlancer.mongodb.gen; - -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.query.MongoDBCreateIndexQuery; - -public final class MongoDBIndexGenerator { - private MongoDBIndexGenerator() { - } - - public static MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { - MongoDBTable randomTable = globalState.getSchema().getRandomTable(); - List columns = Randomly.nonEmptySubset(randomTable.getColumns()); - MongoDBCreateIndexQuery createIndexQuery = new MongoDBCreateIndexQuery(randomTable); - for (MongoDBColumn column : columns) { - createIndexQuery.addIndex(column.getName(), Randomly.getBoolean()); - } - return createIndexQuery; - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java b/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java deleted file mode 100644 index f8b8b3ffd..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBInsertGenerator.java +++ /dev/null @@ -1,38 +0,0 @@ -package sqlancer.mongodb.gen; - -import org.bson.Document; - -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.query.MongoDBInsertQuery; - -public final class MongoDBInsertGenerator { - - private final MongoDBGlobalState globalState; - - private MongoDBInsertGenerator(MongoDBGlobalState globalState) { - this.globalState = globalState; - } - - public static MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { - return new MongoDBInsertGenerator(globalState).generate(); - } - - public MongoDBQueryAdapter generate() { - Document result = new Document(); - MongoDBTable table = globalState.getSchema().getRandomTable(); - MongoDBConstantGenerator constantGenerator = new MongoDBConstantGenerator(globalState); - - for (int i = 0; i < table.getColumns().size(); i++) { - if (!globalState.getDbmsSpecificOptions().testRandomTypes) { - constantGenerator.addRandomConstantWithType(result, table.getColumns().get(i).getName(), - table.getColumns().get(i).getType()); - } else { - constantGenerator.addRandomConstant(result, table.getColumns().get(i).getName()); - } - } - - return new MongoDBInsertQuery(table, result); - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java b/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java deleted file mode 100644 index 3e6c833d3..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBMatchExpressionGenerator.java +++ /dev/null @@ -1,291 +0,0 @@ -package sqlancer.mongodb.gen; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import org.bson.conversions.Bson; - -import com.mongodb.client.model.Filters; - -import sqlancer.Randomly; -import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; -import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; -import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBRegexNode; -import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; -import sqlancer.mongodb.ast.MongoDBUnsupportedPredicate; -import sqlancer.mongodb.test.MongoDBColumnTestReference; -import sqlancer.mongodb.visitor.MongoDBNegateVisitor; - -public class MongoDBMatchExpressionGenerator - extends UntypedExpressionGenerator, MongoDBColumnTestReference> { - - private final MongoDBGlobalState globalState; - - private enum LeafExpression { - BINARY_COMPARISON, REGEX - } - - private enum NonLeafExpression { - BINARY_LOGICAL, UNARY_LOGICAL - } - - public MongoDBMatchExpressionGenerator(MongoDBGlobalState globalState) { - this.globalState = globalState; - } - - @Override - public Node generateLeafNode() { - List possibleOptions = new ArrayList<>(Arrays.asList(LeafExpression.values())); - if (!globalState.getDbmsSpecificOptions().testWithRegex) { - possibleOptions.remove(LeafExpression.REGEX); - } - LeafExpression expr = Randomly.fromList(possibleOptions); - switch (expr) { - case BINARY_COMPARISON: - MongoDBBinaryComparisonOperator operator = MongoDBBinaryComparisonOperator.getRandom(); - MongoDBColumnTestReference reference = (MongoDBColumnTestReference) generateColumn(); - - return new MongoDBBinaryComparisonNode(reference, - generateConstant(reference.getColumnReference().getType()), operator); - case REGEX: - return new MongoDBRegexNode(generateColumn(), - new MongoDBConstantGenerator(globalState).generateConstantWithType(MongoDBDataType.STRING), - getRandomizedRegexOptions()); - default: - throw new AssertionError(); - } - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - - List possibleOptions = new ArrayList<>(Arrays.asList(NonLeafExpression.values())); - NonLeafExpression expr = Randomly.fromList(possibleOptions); - switch (expr) { - case BINARY_LOGICAL: - MongoDBBinaryLogicalOperator binaryOperator = MongoDBBinaryLogicalOperator.getRandom(); - return new MongoDBBinaryLogicalNode(generateExpression(depth + 1), generateExpression(depth + 1), - binaryOperator); - case UNARY_LOGICAL: - MongoDBUnaryLogicalOperator unaryOperator = MongoDBUnaryLogicalOperator.getRandom(); - return new MongoDBUnaryLogicalOperatorNode(generateExpression(depth + 1), unaryOperator); - default: - throw new AssertionError(); - } - } - - @Override - public Node generateConstant() { - MongoDBDataType type = MongoDBDataType.getRandom(globalState); - MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); - if (Randomly.getBooleanWithSmallProbability()) { - return MongoDBConstant.createNullConstant(); - } - return generator.generateConstantWithType(type); - } - - public Node generateConstant(MongoDBDataType type) { - MongoDBConstantGenerator generator = new MongoDBConstantGenerator(globalState); - if (Randomly.getBooleanWithSmallProbability() && !globalState.getDbmsSpecificOptions().nullSafety) { - return MongoDBConstant.createNullConstant(); - } - return generator.generateConstantWithType(type); - } - - private String getRandomizedRegexOptions() { - List s = Randomly.subset("i", "m", "x", "s"); - return String.join("", s); - } - - @Override - protected Node generateColumn() { - return Randomly.fromList(columns); - } - - @Override - public Node generatePredicate() { - Node result = super.generatePredicate(); - return MongoDBNegateVisitor.cleanNegations(result); - } - - @Override - public Node negatePredicate(Node predicate) { - Node result = new MongoDBUnaryLogicalOperatorNode(predicate, - MongoDBUnaryLogicalOperator.NOT); - return MongoDBNegateVisitor.cleanNegations(result); - } - - @Override - public Node isNull(Node expr) { - return new MongoDBUnsupportedPredicate<>(); - } - - public enum MongoDBUnaryLogicalOperator implements Operator { - NOT { - @Override - public Bson applyOperator(Bson inner) { - return Filters.not(inner); - } - - @Override - public String getTextRepresentation() { - return "$not"; - } - }; - - public abstract Bson applyOperator(Bson inner); - - public static MongoDBUnaryLogicalOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum MongoDBBinaryLogicalOperator implements Operator { - AND { - @Override - public Bson applyOperator(Bson left, Bson right) { - return Filters.and(left, right); - } - - @Override - public String getTextRepresentation() { - return "$and"; - } - }, - OR { - @Override - public Bson applyOperator(Bson left, Bson right) { - return Filters.or(left, right); - } - - @Override - public String getTextRepresentation() { - return "$or"; - } - }, - NOR { - @Override - public Bson applyOperator(Bson left, Bson right) { - return Filters.nor(left, right); - } - - @Override - public String getTextRepresentation() { - return "$nor"; - } - }; - - public abstract Bson applyOperator(Bson left, Bson right); - - public static MongoDBBinaryLogicalOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum MongoDBBinaryComparisonOperator implements Operator { - EQUALS { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.eq(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$eq"; - } - }, - NOT_EQUALS { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.ne(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$ne"; - } - }, - GREATER { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.gt(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$gt"; - } - - }, - LESS { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.lt(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$lt"; - } - - }, - GREATER_EQUAL { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.gte(columnName, constant.getValue()); - - } - - @Override - public String getTextRepresentation() { - return "$gte"; - } - - }, - LESS_EQUAL { - @Override - public Bson applyOperator(String columnName, MongoDBConstant constant) { - return Filters.lte(columnName, constant.getValue()); - } - - @Override - public String getTextRepresentation() { - return "$lte"; - } - }; - - public abstract Bson applyOperator(String columnName, MongoDBConstant constant); - - public static MongoDBBinaryComparisonOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum MongoDBRegexOperator implements Operator { - REGEX { - @Override - public Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex, String options) { - return Filters.regex(columnName, regex.getStringValue(), options); - } - - @Override - public String getTextRepresentation() { - return "$regex"; - } - }; - - public abstract Bson applyOperator(String columnName, MongoDBConstant.MongoDBStringConstant regex, - String options); - } -} diff --git a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java b/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java deleted file mode 100644 index 0153a6334..000000000 --- a/src/sqlancer/mongodb/gen/MongoDBTableGenerator.java +++ /dev/null @@ -1,54 +0,0 @@ -package sqlancer.mongodb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; -import sqlancer.mongodb.MongoDBSchema.MongoDBDataType; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.query.MongoDBCreateTableQuery; - -public class MongoDBTableGenerator { - - private MongoDBTable table; - private final List columnsToBeAdded = new ArrayList<>(); - private final MongoDBGlobalState state; - - public MongoDBTableGenerator(MongoDBGlobalState state) { - this.state = state; - } - - public MongoDBQueryAdapter getQuery(MongoDBGlobalState globalState) { - String tableName = globalState.getSchema().getFreeTableName(); - MongoDBCreateTableQuery createTableQuery = new MongoDBCreateTableQuery(tableName); - table = new MongoDBTable(tableName, columnsToBeAdded, false); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - String columnName = String.format("c%d", i); - MongoDBDataType type = createColumn(columnName); - if (globalState.getDbmsSpecificOptions().testValidation) { - createTableQuery.addValidation(columnName, type.getBsonType()); - } - } - globalState.addTable(table); - return createTableQuery; - } - - private MongoDBDataType createColumn(String columnName) { - MongoDBDataType columnType = MongoDBDataType.getRandom(state); - MongoDBColumn newColumn = new MongoDBColumn(columnName, columnType, false, false); - newColumn.setTable(table); - columnsToBeAdded.add(newColumn); - return columnType; - } - - public String getTableName() { - return table.getName(); - } - - public MongoDBTable getGeneratedTable() { - return table; - } -} diff --git a/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java b/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java deleted file mode 100644 index c873b5924..000000000 --- a/src/sqlancer/mongodb/query/MongoDBCreateIndexQuery.java +++ /dev/null @@ -1,77 +0,0 @@ -package sqlancer.mongodb.query; - -import java.util.ArrayList; -import java.util.List; - -import org.bson.conversions.Bson; - -import com.mongodb.client.model.Indexes; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; - -public class MongoDBCreateIndexQuery extends MongoDBQueryAdapter { - - private final MongoDBTable table; - private final List indeces; - private final List logIndeces; - - public MongoDBCreateIndexQuery(MongoDBTable table) { - this.table = table; - this.indeces = new ArrayList<>(); - this.logIndeces = new ArrayList<>(); - } - - public void addIndex(String column, boolean ascending) { - if (ascending) { - indeces.add(Indexes.ascending(column)); - logIndeces.add(column + ": 1"); - } else { - indeces.add(Indexes.descending(column)); - logIndeces.add(column + ": -1"); - } - } - - @Override - public String getLogString() { - StringBuilder sb = new StringBuilder(); - sb.append("db.").append(table.getName()).append(".createIndex({"); - String helper = ""; - for (String index : logIndeces) { - sb.append(helper); - helper = ","; - sb.append(index); - } - sb.append("})\n"); - return sb.toString(); - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - Main.nrSuccessfulActions.addAndGet(1); - Bson index; - if (indeces.size() > 1) { - index = Indexes.compoundIndex(indeces); - } else { - index = indeces.get(0); - } - globalState.getConnection().getDatabase().getCollection(table.getName()).createIndex(index); - return true; - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - -} diff --git a/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java b/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java deleted file mode 100644 index 7bc174c77..000000000 --- a/src/sqlancer/mongodb/query/MongoDBCreateTableQuery.java +++ /dev/null @@ -1,115 +0,0 @@ -package sqlancer.mongodb.query; - -import java.util.ArrayList; -import java.util.List; - -import org.bson.BsonType; -import org.bson.conversions.Bson; - -import com.mongodb.client.model.CreateCollectionOptions; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.ValidationOptions; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; - -public class MongoDBCreateTableQuery extends MongoDBQueryAdapter { - - private final String tableName; - private Bson validationFilter; - private final List logRequiredList; - private final List logPropertiesList; - - public MongoDBCreateTableQuery(String tableName) { - this.tableName = tableName; - this.validationFilter = null; - logRequiredList = new ArrayList<>(); - logPropertiesList = new ArrayList<>(); - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - ValidationOptions collOptions = new ValidationOptions().validator(this.validationFilter); - Main.nrSuccessfulActions.addAndGet(1); - globalState.getConnection().getDatabase().createCollection(tableName, - new CreateCollectionOptions().validationOptions(collOptions)); - return true; - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - String helper = ""; - StringBuilder sb = new StringBuilder(); - sb.append("db.createCollection(\"").append(tableName).append("\", {\n"); - - if (!logPropertiesList.isEmpty()) { - sb.append("validator: {"); - sb.append("$jsonSchema: {"); - sb.append("bsonType:\"object\","); - sb.append("required: [\n"); - for (String req : logRequiredList) { - sb.append(helper); - helper = ","; - sb.append(req); - } - sb.append("],"); - sb.append("properties: {\n"); - for (String prop : logPropertiesList) { - sb.append(prop); - } - sb.append("}}}})"); - } else { - sb.append("})"); - } - - return sb.toString(); - } - - public void addValidation(String columnName, BsonType type) { - Bson nameFilter = Filters.exists(columnName); - Bson typeFilter = Filters.type(columnName, type); - - if (validationFilter == null) { - validationFilter = Filters.and(nameFilter, typeFilter); - } else { - validationFilter = Filters.and(validationFilter, Filters.and(nameFilter, typeFilter)); - } - - logRequiredList.add("\"" + columnName + "\""); - logPropertiesList.add(columnName + ": { bsonType:\"" + bsonTypeToString(type) + "\"},\n"); - } - - public String bsonTypeToString(BsonType type) { - switch (type) { - case DOUBLE: - return "double"; - case STRING: - return "string"; - case BOOLEAN: - return "bool"; - case INT32: - case INT64: - return "int"; - case DATE_TIME: - return "date"; - case TIMESTAMP: - return "timestamp"; - default: - throw new IllegalStateException(); - } - } -} diff --git a/src/sqlancer/mongodb/query/MongoDBInsertQuery.java b/src/sqlancer/mongodb/query/MongoDBInsertQuery.java deleted file mode 100644 index 127dc82d0..000000000 --- a/src/sqlancer/mongodb/query/MongoDBInsertQuery.java +++ /dev/null @@ -1,87 +0,0 @@ -package sqlancer.mongodb.query; - -import org.bson.BsonDateTime; -import org.bson.BsonTimestamp; -import org.bson.Document; -import org.bson.types.ObjectId; - -import com.mongodb.client.result.InsertOneResult; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; - -public class MongoDBInsertQuery extends MongoDBQueryAdapter { - boolean excluded; - private final MongoDBTable table; - private final Document documentToBeInserted; - - public MongoDBInsertQuery(MongoDBTable table, Document documentToBeInserted) { - this.table = table; - this.documentToBeInserted = documentToBeInserted; - this.excluded = false; - } - - @Override - public String getLogString() { - StringBuilder sb = new StringBuilder(); - sb.append("db." + table.getName() + ".insert({"); - String helper = ""; - for (String key : documentToBeInserted.keySet()) { - sb.append(helper); - helper = ", "; - if (documentToBeInserted.get(key) instanceof ObjectId) { - continue; - } - Object value = documentToBeInserted.get(key); - sb.append(key); - sb.append(": "); - sb.append(getStringRepresentation(value)); - } - sb.append("})\n"); - - return sb.toString(); - } - - private String getStringRepresentation(Object value) { - if (value instanceof Double) { - return String.valueOf(value); - } else if (value instanceof Integer) { - return "NumberInt(" + value + ")"; - } else if (value instanceof String) { - return "\"" + value + "\""; - } else if (value instanceof BsonDateTime) { - return "new Date(" + ((BsonDateTime) value).getValue() + ")"; - } else if (value instanceof BsonTimestamp) { - return "Timestamp(" + ((BsonTimestamp) value).getValue() + ",1)"; - } else if (value instanceof Boolean) { - return String.valueOf(value); - } else if (value == null) { - return "null"; - } else { - throw new IllegalStateException(); - } - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - Main.nrSuccessfulActions.addAndGet(1); - InsertOneResult result = globalState.getConnection().getDatabase().getCollection(table.getName()) - .insertOne(documentToBeInserted); - return result.wasAcknowledged(); - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } -} diff --git a/src/sqlancer/mongodb/query/MongoDBRemoveQuery.java b/src/sqlancer/mongodb/query/MongoDBRemoveQuery.java deleted file mode 100644 index 6fe1c9e3f..000000000 --- a/src/sqlancer/mongodb/query/MongoDBRemoveQuery.java +++ /dev/null @@ -1,59 +0,0 @@ -package sqlancer.mongodb.query; - -import org.bson.Document; -import org.bson.types.ObjectId; - -import com.mongodb.client.result.DeleteResult; - -import sqlancer.GlobalState; -import sqlancer.Main; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.MongoDBSchema; - -public class MongoDBRemoveQuery extends MongoDBQueryAdapter { - - private final String objectId; - private final MongoDBSchema.MongoDBTable table; - - public MongoDBRemoveQuery(MongoDBSchema.MongoDBTable table, String objectId) { - this.objectId = objectId; - this.table = table; - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - DeleteResult result = globalState.getConnection().getDatabase().getCollection(table.getName()) - .deleteOne(new Document("_id", new ObjectId(objectId))); - if (result.wasAcknowledged()) { - Main.nrSuccessfulActions.addAndGet(1); - } else { - Main.nrUnsuccessfulActions.addAndGet(1); - } - return result.wasAcknowledged(); - } catch (Exception e) { - Main.nrUnsuccessfulActions.addAndGet(1); - return false; - } - } - - @Override - public ExpectedErrors getExpectedErrors() { - return new ExpectedErrors(); - } - - @Override - public String getLogString() { - StringBuilder stringBuilder = new StringBuilder(); - stringBuilder.append("db.").append(table.getName()).append(".remove({'_id': '").append(objectId).append("'})"); - return stringBuilder.toString(); - } -} diff --git a/src/sqlancer/mongodb/query/MongoDBSelectQuery.java b/src/sqlancer/mongodb/query/MongoDBSelectQuery.java deleted file mode 100644 index 1288f114c..000000000 --- a/src/sqlancer/mongodb/query/MongoDBSelectQuery.java +++ /dev/null @@ -1,147 +0,0 @@ -package sqlancer.mongodb.query; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.bson.Document; -import org.bson.conversions.Bson; - -import com.mongodb.client.MongoCollection; -import com.mongodb.client.MongoCursor; - -import sqlancer.GlobalState; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.mongodb.MongoDBConnection; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.visitor.MongoDBVisitor; - -public class MongoDBSelectQuery extends MongoDBQueryAdapter { - - private final MongoDBSelect select; - - private List resultSet; - - public MongoDBSelectQuery(MongoDBSelect select) { - this.select = select; - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - throw new UnsupportedOperationException(); - } - - @Override - public ExpectedErrors getExpectedErrors() { - ExpectedErrors errors = new ExpectedErrors(); - // ARITHMETIC - errors.add("Failed to optimize pipeline :: caused by :: Can't coerce out of range value"); - errors.add("Can't coerce out of range value"); - errors.add("date overflow in $add"); - errors.add("Failed to optimize pipeline :: caused by :: $sqrt only supports numeric types, not"); - errors.add("Failed to optimize pipeline :: caused by :: $sqrt's argument must be greater than or equal to 0"); - errors.add("Failed to optimize pipeline :: caused by :: $pow's base must be numeric, not"); - errors.add("Failed to optimize pipeline :: caused by :: $pow cannot take a base of 0 and a negative exponent"); - errors.add("Failed to optimize pipeline :: caused by :: $add only supports numeric or date types, not"); - errors.add("Failed to optimize pipeline :: caused by :: $exp only supports numeric types, not"); - errors.add("Failed to optimize pipeline :: caused by :: $log's base must be numeric, not"); - errors.add("Failed to optimize pipeline :: caused by :: $log's base must be a positive number not equal to 1"); - errors.add("Failed to optimize pipeline :: caused by :: $multiply only supports numeric types, not"); - errors.add("$log's argument must be numeric, not"); - errors.add("$log's argument must be a positive number, but"); - errors.add("$log's base must be numeric, not"); - errors.add("$log's base must be a positive number not equal to 1"); - errors.add("$divide only supports numeric types, not"); - errors.add("can't $divide by zero"); - errors.add("$pow's exponent must be numeric, not"); - errors.add("$pow's base must be numeric, not"); - errors.add("$pow cannot take a base of 0 and a negative exponent"); - errors.add("$add only supports numeric or date types, not"); - errors.add("only one date allowed in an $add expression"); - errors.add("$multiply only supports numeric types, not"); - errors.add("$exp only supports numeric types, not"); - errors.add("$sqrt's argument must be greater than or equal to 0"); - errors.add("$sqrt only supports numeric types, not"); - - // REGEX - errors.add("Regular expression is invalid: nothing to repeat"); - errors.add("Regular expression is invalid: missing terminating ] for character class"); - errors.add("Regular expression is invalid: unmatched parentheses"); - errors.add("Regular expression is invalid: missing )"); - errors.add("Regular expression is invalid: invalid UTF-8 string"); - errors.add("Regular expression is invalid: \\k is not followed by a braced, angle-bracketed, or quoted name"); - errors.add("Regular expression is invalid: missing opening brace after \\\\o"); - errors.add("Regular expression is invalid: reference to non-existent subpattern"); - errors.add("Regular expression is invalid: \\ at end of pattern"); - errors.add("Regular expression is invalid: PCRE does not support \\L, \\l, \\N{name}, \\U, or \\u"); - errors.add("Regular expression is invalid: (?R or (?[+-]digits must be followed by )"); - errors.add("Regular expression is invalid: unknown property name after \\P or \\p"); - errors.add("Regular expression is invalid: (*VERB) not recognized or malformed"); - errors.add("Regular expression is invalid: a numbered reference must not be zero"); - errors.add("Regular expression is invalid: unrecognized character after (? or (?-"); - errors.add("Regular expression is invalid: \\c at end of pattern"); - errors.add("Regular expression is invalid: malformed \\P or \\p sequence"); - errors.add("Regular expression is invalid: range out of order in character class"); - errors.add("Regular expression is invalid: group name must start with a non-digit"); - errors.add("Regular expression is invalid: \\c must be followed by an ASCII character"); - errors.add("Regular expression is invalid: subpattern name expected"); - errors.add("Regular expression is invalid: POSIX collating elements are not supported"); - errors.add("Regular expression is invalid: closing ) for (?C expected"); - errors.add("Regular expression is invalid: syntax error in subpattern name (missing terminator)"); - errors.add("Regular expression is invalid: \\\\N is not supported in a class"); - errors.add("Regular expression is invalid: non-octal character in \\o{} (closing brace missing?)"); - errors.add("Regular expression is invalid: non-hex character in \\x{} (closing brace missing?)"); - errors.add( - "Regular expression is invalid: \\g is not followed by a braced, angle-bracketed, or quoted name/number or by a plain number"); - errors.add("Regular expression is invalid: digits missing in \\x{} or \\o{}"); - errors.add("Regular expression is invalid: malformed number or name after (?("); - errors.add("Regular expression is invalid: digit expected after (?+"); - errors.add("Regular expression is invalid: assertion expected after (?( or (?(?C)"); - errors.add("Regular expression is invalid: unrecognized character after (?P"); - - return errors; - } - - @Override - public > SQLancerResultSet executeAndGet(G globalState, - String... fills) throws Exception { - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(this.getLogString()); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - List pipeline = MongoDBVisitor.asQuery(select); - - MongoCollection collection = globalState.getConnection().getDatabase() - .getCollection(select.getMainTableName()); - MongoCursor cursor = collection.aggregate(pipeline).cursor(); - resultSet = new ArrayList<>(); - while (cursor.hasNext()) { - Document document = cursor.next(); - resultSet.add(document); - } - return null; - } - - @Override - public String getLogString() { - return MongoDBVisitor.asStringLog(select); - } - - public List getResultSet() { - return resultSet; - } - -} diff --git a/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java b/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java deleted file mode 100644 index 59a2a6724..000000000 --- a/src/sqlancer/mongodb/test/MongoDBColumnTestReference.java +++ /dev/null @@ -1,40 +0,0 @@ -package sqlancer.mongodb.test; - -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; -import sqlancer.mongodb.ast.MongoDBExpression; - -public class MongoDBColumnTestReference implements Node { - - private final MongoDBColumn columnReference; - private final boolean inMainTable; - - public MongoDBColumnTestReference(MongoDBColumn columnReference, boolean inMainTable) { - this.columnReference = columnReference; - this.inMainTable = inMainTable; - } - - public String getQueryString() { - if (inMainTable) { - return this.columnReference.getName(); - } else { - return "join_" + this.columnReference.getTable().getName() + "." + this.columnReference.getName(); - } - } - - public boolean inMainTable() { - return inMainTable; - } - - public String getTableName() { - return this.columnReference.getTable().getName(); - } - - public String getPlainName() { - return this.columnReference.getName(); - } - - public MongoDBColumn getColumnReference() { - return columnReference; - } -} diff --git a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java deleted file mode 100644 index bf39643c1..000000000 --- a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalBase.java +++ /dev/null @@ -1,90 +0,0 @@ -package sqlancer.mongodb.test; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.DocumentRemovalOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mongodb.MongoDBProvider; -import sqlancer.mongodb.MongoDBSchema; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; - -public class MongoDBDocumentRemovalBase - extends DocumentRemovalOracleBase, MongoDBProvider.MongoDBGlobalState> - implements TestOracle { - - protected MongoDBSchema schema; - protected MongoDBSchema.MongoDBTables targetTables; - protected MongoDBSchema.MongoDBTable mainTable; - protected List targetColumns; - protected MongoDBMatchExpressionGenerator expressionGenerator; - protected MongoDBSelect select; - - protected MongoDBDocumentRemovalBase(MongoDBProvider.MongoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getRandomTableNonEmptyTables(); - mainTable = targetTables.getTables().get(0); - generateTargetColumns(); - expressionGenerator = new MongoDBMatchExpressionGenerator(state).setColumns(targetColumns); - initializeDocumentRemovalOracle(); - select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); - select.setProjectionList(targetColumns); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setLookupList(targetColumns); - } else { - select.setLookupList(Randomly.nonEmptySubset(targetColumns)); - } - if (state.getDbmsSpecificOptions().testComputedValues) { - generateComputedColumns(); - } - } - - private void generateTargetColumns() { - targetColumns = new ArrayList<>(); - for (MongoDBSchema.MongoDBColumn c : mainTable.getColumns()) { - targetColumns.add(new MongoDBColumnTestReference(c, true)); - } - List joinsOtherTables = new ArrayList<>(); - if (!state.getDbmsSpecificOptions().nullSafety) { - for (int i = 1; i < targetTables.getTables().size(); i++) { - MongoDBSchema.MongoDBTable procTable = targetTables.getTables().get(i); - for (MongoDBSchema.MongoDBColumn c : procTable.getColumns()) { - joinsOtherTables.add(new MongoDBColumnTestReference(c, false)); - } - } - } - if (!joinsOtherTables.isEmpty()) { - int randNumber = state.getRandomly().getInteger(1, Math.min(joinsOtherTables.size(), 4)); - List subsetJoinsOtherTables = Randomly.nonEmptySubset(joinsOtherTables, - randNumber); - targetColumns.addAll(subsetJoinsOtherTables); - } - } - - private void generateComputedColumns() { - List> computedColumns = new ArrayList<>(); - int numberComputedColumns = state.getRandomly().getInteger(1, 4); - MongoDBComputedExpressionGenerator generator = new MongoDBComputedExpressionGenerator(state) - .setColumns(targetColumns); - for (int i = 0; i < numberComputedColumns; i++) { - computedColumns.add(generator.generateExpression()); - } - select.setComputedClause(computedColumns); - } - - @Override - protected ExpressionGenerator> getGen() { - return expressionGenerator; - } -} diff --git a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java b/src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java deleted file mode 100644 index ece3193d1..000000000 --- a/src/sqlancer/mongodb/test/MongoDBDocumentRemovalTester.java +++ /dev/null @@ -1,49 +0,0 @@ -package sqlancer.mongodb.test; - -import static sqlancer.mongodb.MongoDBComparatorHelper.getResultSetAsDocumentList; - -import java.util.List; - -import org.bson.Document; - -import sqlancer.Randomly; -import sqlancer.mongodb.MongoDBProvider; -import sqlancer.mongodb.MongoDBQueryAdapter; -import sqlancer.mongodb.gen.MongoDBInsertGenerator; -import sqlancer.mongodb.query.MongoDBRemoveQuery; -import sqlancer.mongodb.query.MongoDBSelectQuery; - -public class MongoDBDocumentRemovalTester extends MongoDBDocumentRemovalBase { - public MongoDBDocumentRemovalTester(MongoDBProvider.MongoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - - select.setWithCountClause(false); - - select.setFilterClause(predicate); - MongoDBSelectQuery selectQuery = new MongoDBSelectQuery(select); - List firstResultSet = getResultSetAsDocumentList(selectQuery, state); - if (firstResultSet == null || firstResultSet.isEmpty()) { - return; - } - - Document documentToRemove = Randomly.fromList(firstResultSet); - MongoDBRemoveQuery removeQuery = new MongoDBRemoveQuery(mainTable, documentToRemove.get("_id").toString()); - state.executeStatement(removeQuery); - - selectQuery = new MongoDBSelectQuery(select); - List secondResultSet = getResultSetAsDocumentList(selectQuery, state); - - MongoDBQueryAdapter insertQuery = MongoDBInsertGenerator.getQuery(state); - state.executeStatement(insertQuery); - - if (secondResultSet.size() + 1 != firstResultSet.size()) { - String assertMessage = "The Result Sizes mismatches!"; - throw new AssertionError(assertMessage); - } - } -} diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java deleted file mode 100644 index ea6d39bbf..000000000 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningBase.java +++ /dev/null @@ -1,93 +0,0 @@ -package sqlancer.mongodb.test; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.MongoDBSchema; -import sqlancer.mongodb.MongoDBSchema.MongoDBColumn; -import sqlancer.mongodb.MongoDBSchema.MongoDBTable; -import sqlancer.mongodb.MongoDBSchema.MongoDBTables; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; - -public class MongoDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, MongoDBGlobalState> - implements TestOracle { - - protected MongoDBSchema schema; - protected MongoDBTables targetTables; - protected MongoDBTable mainTable; - protected List targetColumns; - protected MongoDBMatchExpressionGenerator expressionGenerator; - protected MongoDBSelect select; - - public MongoDBQueryPartitioningBase(MongoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getRandomTableNonEmptyTables(); - mainTable = targetTables.getTables().get(0); - generateTargetColumns(); - expressionGenerator = new MongoDBMatchExpressionGenerator(state).setColumns(targetColumns); - initializeTernaryPredicateVariants(); - select = new MongoDBSelect<>(mainTable.getName(), targetColumns.get(0)); - select.setProjectionList(targetColumns); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setLookupList(targetColumns); - } else { - select.setLookupList(Randomly.nonEmptySubset(targetColumns)); - } - if (state.getDbmsSpecificOptions().testComputedValues) { - generateComputedColumns(); - } - } - - private void generateComputedColumns() { - List> computedColumns = new ArrayList<>(); - int numberComputedColumns = state.getRandomly().getInteger(1, 4); - MongoDBComputedExpressionGenerator generator = new MongoDBComputedExpressionGenerator(state) - .setColumns(targetColumns); - for (int i = 0; i < numberComputedColumns; i++) { - computedColumns.add(generator.generateExpression()); - } - select.setComputedClause(computedColumns); - } - - private void generateTargetColumns() { - targetColumns = new ArrayList<>(); - for (MongoDBColumn c : mainTable.getColumns()) { - targetColumns.add(new MongoDBColumnTestReference(c, true)); - } - List joinsOtherTables = new ArrayList<>(); - if (!state.getDbmsSpecificOptions().nullSafety) { - for (int i = 1; i < targetTables.getTables().size(); i++) { - MongoDBTable procTable = targetTables.getTables().get(i); - for (MongoDBColumn c : procTable.getColumns()) { - joinsOtherTables.add(new MongoDBColumnTestReference(c, false)); - } - } - } - if (!joinsOtherTables.isEmpty()) { - int randNumber = state.getRandomly().getInteger(1, Math.min(joinsOtherTables.size(), 4)); - List subsetJoinsOtherTables = Randomly.nonEmptySubset(joinsOtherTables, - randNumber); - targetColumns.addAll(subsetJoinsOtherTables); - } - } - - @Override - protected ExpressionGenerator> getGen() { - return expressionGenerator; - } -} diff --git a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java b/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java deleted file mode 100644 index 53f625d9b..000000000 --- a/src/sqlancer/mongodb/test/MongoDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,48 +0,0 @@ -package sqlancer.mongodb.test; - -import static sqlancer.mongodb.MongoDBComparatorHelper.getResultSetAsDocumentList; - -import java.util.List; - -import org.bson.Document; - -import sqlancer.mongodb.MongoDBComparatorHelper; -import sqlancer.mongodb.MongoDBProvider.MongoDBGlobalState; -import sqlancer.mongodb.query.MongoDBSelectQuery; - -public class MongoDBQueryPartitioningWhereTester extends MongoDBQueryPartitioningBase { - public MongoDBQueryPartitioningWhereTester(MongoDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - - select.setWithCountClause(false); - - select.setFilterClause(null); - MongoDBSelectQuery q = new MongoDBSelectQuery(select); - List firstResultSet = getResultSetAsDocumentList(q, state); - - select.setFilterClause(predicate); - q = new MongoDBSelectQuery(select); - List secondResultSet = getResultSetAsDocumentList(q, state); - - select.setFilterClause(negatedPredicate); - q = new MongoDBSelectQuery(select); - List thirdResultSet = getResultSetAsDocumentList(q, state); - - if (state.getDbmsSpecificOptions().testWithCount) { - select.setWithCountClause(true); - select.setFilterClause(predicate); - q = new MongoDBSelectQuery(select); - List forthResultSet = getResultSetAsDocumentList(q, state); - MongoDBComparatorHelper.assumeCountIsEqual(secondResultSet, forthResultSet, q); - } - - secondResultSet.addAll(thirdResultSet); - MongoDBComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, q); - - } -} diff --git a/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java deleted file mode 100644 index 39b607f67..000000000 --- a/src/sqlancer/mongodb/visitor/MongoDBNegateVisitor.java +++ /dev/null @@ -1,161 +0,0 @@ -package sqlancer.mongodb.visitor; - -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator.AND; -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator.NOR; -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBBinaryLogicalOperator.OR; -import static sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator.MongoDBUnaryLogicalOperator.NOT; - -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; -import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBRegexNode; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; -import sqlancer.mongodb.gen.MongoDBMatchExpressionGenerator; - -public class MongoDBNegateVisitor extends MongoDBVisitor { - - private boolean negate; - Node negatedExpression; - - public MongoDBNegateVisitor(boolean negate) { - this.negate = negate; - } - - @Override - public void visit(Node expr) { - if (expr instanceof MongoDBConstant) { - visit((MongoDBConstant) expr); - } else if (expr instanceof MongoDBSelect) { - visit((MongoDBSelect) expr); - } else if (expr instanceof MongoDBBinaryComparisonNode) { - visit((MongoDBBinaryComparisonNode) expr); - } else if (expr instanceof MongoDBUnaryLogicalOperatorNode) { - visit((MongoDBUnaryLogicalOperatorNode) expr); - } else if (expr instanceof MongoDBRegexNode) { - visit((MongoDBRegexNode) expr); - } else if (expr instanceof MongoDBBinaryLogicalNode) { - visit((MongoDBBinaryLogicalNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public void visit(MongoDBBinaryComparisonNode expr) { - - if (negate) { - negatedExpression = new MongoDBUnaryLogicalOperatorNode(expr, NOT); - switch (expr.operator()) { - case EQUALS: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.NOT_EQUALS); - break; - case NOT_EQUALS: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.EQUALS); - break; - case LESS: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER_EQUAL); - break; - case LESS_EQUAL: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.GREATER); - break; - case GREATER: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS_EQUAL); - break; - case GREATER_EQUAL: - negatedExpression = new MongoDBBinaryComparisonNode(expr.getLeft(), expr.getRight(), - MongoDBMatchExpressionGenerator.MongoDBBinaryComparisonOperator.LESS); - break; - default: - throw new UnsupportedOperationException(); - } - } else { - negatedExpression = expr; - } - } - - public void visit(MongoDBRegexNode expr) { - if (negate) { - negatedExpression = new MongoDBUnaryLogicalOperatorNode(expr, NOT); - } else { - negatedExpression = expr; - } - } - - public void visit(MongoDBUnaryLogicalOperatorNode expr) { - if (!(expr.operator().equals(NOT))) { - throw new UnsupportedOperationException(); - } - negate = !negate; - visit(expr.getExpr()); - } - - public void visit(MongoDBBinaryLogicalNode expr) { - boolean saveNegate = negate; - Node left; - Node right; - switch (expr.operator()) { - case OR: - negate = false; - visit(expr.getLeft()); - left = negatedExpression; - negate = false; - visit(expr.getRight()); - right = negatedExpression; - if (saveNegate) { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, NOR); - } else { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, OR); - } - break; - case AND: - negate = saveNegate; - visit(expr.getLeft()); - left = negatedExpression; - negate = saveNegate; - visit(expr.getRight()); - right = negatedExpression; - if (saveNegate) { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, OR); - } else { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, AND); - } - break; - case NOR: - negate = false; - visit(expr.getLeft()); - left = negatedExpression; - negate = false; - visit(expr.getRight()); - right = negatedExpression; - if (saveNegate) { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, OR); - } else { - negatedExpression = new MongoDBBinaryLogicalNode(left, right, NOR); - } - break; - default: - throw new UnsupportedOperationException(expr.getOperatorRepresentation()); - } - } - - @Override - public void visit(MongoDBConstant c) { - negatedExpression = c; - } - - @Override - public void visit(MongoDBSelect s) { - throw new UnsupportedOperationException(); - } - - public Node getNegatedExpression() { - return negatedExpression; - } -} diff --git a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java deleted file mode 100644 index 4c55e17a6..000000000 --- a/src/sqlancer/mongodb/visitor/MongoDBToLogVisitor.java +++ /dev/null @@ -1,195 +0,0 @@ -package sqlancer.mongodb.visitor; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; -import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBRegexNode; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; -import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator.ComputedFunction; -import sqlancer.mongodb.test.MongoDBColumnTestReference; - -public class MongoDBToLogVisitor extends MongoDBVisitor { - - private String mainTableName; - private List lookups; - private String filter; - private String projects; - private boolean hasFilter; - private boolean withCount; - - public String visitLog(Node expr) { - if (expr instanceof MongoDBUnaryLogicalOperatorNode) { - return visit((MongoDBUnaryLogicalOperatorNode) expr); - } else if (expr instanceof MongoDBBinaryLogicalNode) { - return visit((MongoDBBinaryLogicalNode) expr); - } else if (expr instanceof MongoDBBinaryComparisonNode) { - return visit((MongoDBBinaryComparisonNode) expr); - } else if (expr instanceof MongoDBRegexNode) { - return visit((MongoDBRegexNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public String visitComputed(Node expr) { - if (expr instanceof NewFunctionNode) { - return visitComputed((NewFunctionNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public String visitComputed(NewFunctionNode expr) { - List arguments = new ArrayList<>(); - for (int i = 0; i < expr.getArgs().size(); i++) { - if (expr.getArgs().get(i) instanceof MongoDBConstant) { - arguments.add(((MongoDBConstant) expr.getArgs().get(i)).getLogValue()); - continue; - } - if (expr.getArgs().get(i) instanceof MongoDBColumnTestReference) { - arguments.add("\"$" + ((MongoDBColumnTestReference) expr.getArgs().get(i)).getQueryString() + "\""); - continue; - } - if (expr.getArgs().get(i) instanceof NewFunctionNode) { - arguments.add(visitComputed((NewFunctionNode) expr.getArgs().get(i))); - } else { - throw new AssertionError(); - } - } - if (!(expr.getFunc() instanceof ComputedFunction)) { - throw new AssertionError(expr.getClass()); - } - - StringBuilder sb = new StringBuilder(); - sb.append("{"); - sb.append(((ComputedFunction) expr.getFunc()).getOperator()); - sb.append(": ["); - String helper = ""; - for (String arg : arguments) { - sb.append(helper); - helper = ", "; - sb.append(arg); - } - sb.append("]}"); - return sb.toString(); - } - - public String visit(MongoDBUnaryLogicalOperatorNode expr) { - String inner = visitLog(expr.getExpr()); - return "{ " + expr.operator().getTextRepresentation() + ": [" + inner + "]}"; - } - - public String visit(MongoDBBinaryLogicalNode expr) { - String left = visitLog(expr.getLeft()); - String right = visitLog(expr.getRight()); - - return "{" + expr.operator().getTextRepresentation() + ":[" + left + "," + right + "]}"; - } - - public String visit(MongoDBBinaryComparisonNode expr) { - Node left = expr.getLeft(); - Node right = expr.getRight(); - assert left instanceof MongoDBColumnTestReference; - assert right instanceof MongoDBConstant; - - return "{\"" + ((MongoDBColumnTestReference) left).getQueryString() + "\": {" - + expr.operator().getTextRepresentation() + ": " + ((MongoDBConstant) right).getLogValue() + "}}"; - } - - public String visit(MongoDBRegexNode expr) { - Node left = expr.getLeft(); - Node right = expr.getRight(); - - return "{\"" + ((MongoDBColumnTestReference) left).getQueryString() + "\": {" - + expr.operator().getTextRepresentation() + ": \'" - + ((MongoDBConstant.MongoDBStringConstant) right).getStringValue() + "\', $options: \'" - + expr.getOptions() + "\'}}"; - } - - @Override - public void visit(MongoDBConstant c) { - throw new UnsupportedOperationException(); - } - - @Override - public void visit(MongoDBSelect select) { - hasFilter = select.hasFilter(); - mainTableName = select.getMainTableName(); - setLookups(select); - if (hasFilter) { - setFilter(select); - } - setProjects(select); - withCount = select.getWithCountClause(); - } - - private void setFilter(MongoDBSelect select) { - filter = visitLog(select.getFilterClause()); - } - - private void setLookups(MongoDBSelect select) { - lookups = new ArrayList<>(); - for (MongoDBColumnTestReference testReference : select.getLookupList()) { - if (testReference.inMainTable()) { - continue; - } - String newLookup = "{ $lookup: { from: \"" + testReference.getTableName() + "\", localField: \"" - + select.getJoinColumn().getPlainName() + "\", foreignField: \"" + testReference.getPlainName() - + "\", as: \"" + testReference.getQueryString() + "\"}},\n"; - lookups.add(newLookup); - } - } - - private void setProjects(MongoDBSelect select) { - StringBuilder sb = new StringBuilder(); - sb.append("{"); - String helper = ""; - for (MongoDBColumnTestReference reference : select.getProjectionList()) { - sb.append(helper); - helper = ","; - sb.append("\"").append(reference.getQueryString()).append("\"").append(": 1"); - } - sb.append("\n"); - if (select.hasComputed()) { - String name = "computed"; - int number = 0; - for (Node expressionNode : select.getComputedClause()) { - sb.append(helper); - helper = ",\n"; - sb.append("\"" + name + number + "\": " + visitComputed(expressionNode)); - number++; - } - } - sb.append("}"); - projects = sb.toString(); - } - - public String getStringLog() { - StringBuilder sb = new StringBuilder(); - sb.append("db.").append(mainTableName).append(".aggregate([\n"); - for (String lookup : lookups) { - sb.append(lookup); - } - if (hasFilter) { - sb.append("{ $match: "); - sb.append(filter); - sb.append("},\n"); - } - sb.append("{ $project : "); - sb.append(projects); - sb.append("}"); - if (withCount) { - sb.append(",\n"); - sb.append(" {$count: \"count\"}\n"); - } - sb.append("])\n"); - return sb.toString(); - } -} diff --git a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java deleted file mode 100644 index 8efbf97e4..000000000 --- a/src/sqlancer/mongodb/visitor/MongoDBToQueryVisitor.java +++ /dev/null @@ -1,184 +0,0 @@ -package sqlancer.mongodb.visitor; - -import static com.mongodb.client.model.Aggregates.match; -import static com.mongodb.client.model.Aggregates.project; -import static com.mongodb.client.model.Projections.fields; -import static com.mongodb.client.model.Projections.include; - -import java.io.Serializable; -import java.util.ArrayList; -import java.util.List; - -import org.bson.Document; -import org.bson.conversions.Bson; - -import com.mongodb.client.model.Aggregates; -import com.mongodb.client.model.Projections; - -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.ast.MongoDBBinaryComparisonNode; -import sqlancer.mongodb.ast.MongoDBBinaryLogicalNode; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBConstant.MongoDBStringConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBRegexNode; -import sqlancer.mongodb.ast.MongoDBSelect; -import sqlancer.mongodb.ast.MongoDBUnaryLogicalOperatorNode; -import sqlancer.mongodb.gen.MongoDBComputedExpressionGenerator.ComputedFunction; -import sqlancer.mongodb.test.MongoDBColumnTestReference; - -public class MongoDBToQueryVisitor extends MongoDBVisitor { - - private List lookup; - private Bson filter; - private Bson projection; - private Bson count; - private boolean hasFilter; - private boolean hasCountClause; - - public Bson visitBson(Node expr) { - if (expr instanceof MongoDBUnaryLogicalOperatorNode) { - return visit((MongoDBUnaryLogicalOperatorNode) expr); - } else if (expr instanceof MongoDBBinaryLogicalNode) { - return visit((MongoDBBinaryLogicalNode) expr); - } else if (expr instanceof MongoDBBinaryComparisonNode) { - return visit((MongoDBBinaryComparisonNode) expr); - } else if (expr instanceof MongoDBRegexNode) { - return visit((MongoDBRegexNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public Document visitComputed(Node expr) { - if (expr instanceof NewFunctionNode) { - return visitComputed((NewFunctionNode) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public Document visitComputed(NewFunctionNode expr) { - List visitedArgs = new ArrayList<>(); - for (int i = 0; i < expr.getArgs().size(); i++) { - if (expr.getArgs().get(i) instanceof MongoDBConstant) { - visitedArgs.add(((MongoDBConstant) expr.getArgs().get(i)).getSerializedValue()); - continue; - } - if (expr.getArgs().get(i) instanceof MongoDBColumnTestReference) { - visitedArgs.add("$" + ((MongoDBColumnTestReference) expr.getArgs().get(i)).getQueryString()); - continue; - } - if (expr.getArgs().get(i) instanceof NewFunctionNode) { - visitedArgs.add(visitComputed((NewFunctionNode) expr.getArgs().get(i))); - } else { - throw new AssertionError(); - } - } - if (expr.getFunc() instanceof ComputedFunction) { - return new Document(((ComputedFunction) expr.getFunc()).getOperator(), visitedArgs); - } else { - throw new AssertionError(expr.getClass()); - } - - } - - public Bson visit(MongoDBUnaryLogicalOperatorNode expr) { - Bson inner = visitBson(expr.getExpr()); - return expr.operator().applyOperator(inner); - } - - public Bson visit(MongoDBBinaryLogicalNode expr) { - Bson left = visitBson(expr.getLeft()); - Bson right = visitBson(expr.getRight()); - return expr.operator().applyOperator(left, right); - } - - public Bson visit(MongoDBRegexNode expr) { - Node left = expr.getLeft(); - Node right = expr.getRight(); - - String columnName = ((MongoDBColumnTestReference) left).getQueryString(); - - return expr.operator().applyOperator(columnName, (MongoDBStringConstant) right, expr.getOptions()); - } - - public Bson visit(MongoDBBinaryComparisonNode expr) { - Node left = expr.getLeft(); - Node right = expr.getRight(); - assert left instanceof MongoDBColumnTestReference; - assert right instanceof MongoDBConstant; - - String columnName = ((MongoDBColumnTestReference) left).getQueryString(); - return expr.operator().applyOperator(columnName, (MongoDBConstant) right); - } - - @Override - public void visit(MongoDBConstant c) { - throw new UnsupportedOperationException(); - } - - @Override - public void visit(MongoDBSelect select) { - hasFilter = select.hasFilter(); - setLookup(select); - if (hasFilter) { - setFilter(select); - } - setProjection(select); - hasCountClause = select.getWithCountClause(); - if (hasCountClause) { - setCount(); - } - } - - private void setCount() { - count = Aggregates.count("count"); - } - - private void setFilter(MongoDBSelect select) { - filter = match(this.visitBson(select.getFilterClause())); - } - - private void setLookup(MongoDBSelect select) { - lookup = new ArrayList<>(); - for (MongoDBColumnTestReference reference : select.getLookupList()) { - if (reference.inMainTable()) { - continue; - } - lookup.add(Aggregates.lookup(reference.getTableName(), select.getJoinColumn().getPlainName(), - reference.getPlainName(), reference.getQueryString())); - } - } - - private void setProjection(MongoDBSelect select) { - List stringProjects = new ArrayList<>(); - for (MongoDBColumnTestReference ref : select.getProjectionList()) { - stringProjects.add(ref.getQueryString()); - } - List projections = new ArrayList<>(); - projections.add(include(stringProjects)); - if (select.hasComputed()) { - String name = "computed"; - int number = 0; - for (Node expressionNode : select.getComputedClause()) { - projections.add(Projections.computed(name + number, visitComputed(expressionNode))); - number++; - } - } - projection = project(fields(projections)); - } - - public List getPipeline() { - List result = new ArrayList<>(lookup); - if (hasFilter) { - result.add(filter); - } - result.add(projection); - if (hasCountClause) { - result.add(count); - } - return result; - } -} diff --git a/src/sqlancer/mongodb/visitor/MongoDBVisitor.java b/src/sqlancer/mongodb/visitor/MongoDBVisitor.java deleted file mode 100644 index e02a50f02..000000000 --- a/src/sqlancer/mongodb/visitor/MongoDBVisitor.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.mongodb.visitor; - -import java.util.List; - -import org.bson.conversions.Bson; - -import sqlancer.common.ast.newast.Node; -import sqlancer.mongodb.ast.MongoDBConstant; -import sqlancer.mongodb.ast.MongoDBExpression; -import sqlancer.mongodb.ast.MongoDBSelect; - -public abstract class MongoDBVisitor { - - public abstract void visit(MongoDBConstant c); - - public abstract void visit(MongoDBSelect s); - - public void visit(Node expr) { - if (expr instanceof MongoDBConstant) { - visit((MongoDBConstant) expr); - } else if (expr instanceof MongoDBSelect) { - visit((MongoDBSelect) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - public static List asQuery(Node expr) { - MongoDBToQueryVisitor visitor = new MongoDBToQueryVisitor(); - visitor.visit(expr); - return visitor.getPipeline(); - } - - public static String asStringLog(Node expr) { - MongoDBToLogVisitor visitor = new MongoDBToLogVisitor(); - visitor.visit(expr); - return visitor.getStringLog(); - } - - public static Node cleanNegations(Node expr) { - MongoDBNegateVisitor visitor = new MongoDBNegateVisitor(false); - visitor.visit(expr); - return visitor.getNegatedExpression(); - } -} From d01c4a5ddff7d8ae0250a02322ab539526ba7f41 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 17 Mar 2024 10:32:26 +0800 Subject: [PATCH 403/856] Support TiFlash for TiDB (#919) --- src/sqlancer/tidb/TiDBOptions.java | 3 +++ src/sqlancer/tidb/TiDBProvider.java | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 7ff20027d..d35ce9656 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -36,6 +36,9 @@ public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = "--enable-non-prepared-plan-cache") public boolean nonPreparePlanCache; + @Parameter(names = { "--tiflash" }, description = "Enable TiFlash") + public boolean tiflash; + public enum TiDBOracleFactory implements OracleFactory { HAVING { @Override diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index c5c334f11..05ef0d9c6 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -154,6 +154,21 @@ public void generateDatabase(TiDBGlobalState globalState) throws Exception { } } } + + // TiFlash replication settings + if (globalState.getDbmsSpecificOptions().tiflash) { + ExpectedErrors errors = new ExpectedErrors(); + TiDBErrors.addExpressionErrors(errors); + for (TiDBTable table : globalState.getSchema().getDatabaseTables()) { + if (!table.isView()) { + globalState.executeStatement( + new SQLQueryAdapter("ALTER TABLE " + table.getName() + " SET TIFLASH REPLICA 1;", errors)); + } + } + if (Randomly.getBoolean()) { + globalState.executeStatement(new SQLQueryAdapter("set @@tidb_enforce_mpp=1;")); + } + } } @Override From 49afc88429744b54a7bc25d6006d63598f08da00 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Sun, 17 Mar 2024 11:26:50 +0800 Subject: [PATCH 404/856] TiDB DQP (#920) --- src/sqlancer/tidb/TiDBBugs.java | 15 ++++ src/sqlancer/tidb/TiDBErrors.java | 5 +- src/sqlancer/tidb/TiDBOptions.java | 7 ++ .../tidb/gen/TiDBAlterTableGenerator.java | 1 + src/sqlancer/tidb/gen/TiDBHintGenerator.java | 55 +++++++++++-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 4 +- src/sqlancer/tidb/oracle/TiDBDQPOracle.java | 80 +++++++++++++++++++ test/sqlancer/dqp/tidb/TestTiDBDQP.java | 22 +++++ 8 files changed, 182 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/tidb/oracle/TiDBDQPOracle.java create mode 100644 test/sqlancer/dqp/tidb/TestTiDBDQP.java diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index 4aef122e3..81f27b705 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -17,6 +17,21 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/44747 public static boolean bug44747 = true; + // https://github.com/pingcap/tidb/issues/46556 + public static boolean bug46556 = true; + + // https://github.com/pingcap/tidb/issues/46591 + public static boolean bug46591 = true; + + // https://github.com/pingcap/tidb/issues/46598 + public static boolean bug46598 = true; + + // https://github.com/pingcap/tidb/issues/47346 + public static boolean bug47346 = true; + + // https://github.com/pingcap/tidb/issues/47348 + public static boolean bug47348 = true; + private TiDBBugs() { } diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 2068f9f1e..fd4d23d3d 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -25,9 +25,12 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("is not valid for CHARACTER SET"); errors.add("DOUBLE value is out of range"); - errors.add("Data truncation: %s value is out of range in '%s'"); + errors.add("Data truncat"); errors.add("Truncated incorrect FLOAT value"); errors.add("Bad Number"); + errors.add("strconv.Atoi: parsing"); + errors.add("expected integer"); + errors.add("Duplicate entry"); // regex errors.add("error parsing regexp"); diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index d35ce9656..6c69e84a1 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -15,6 +15,7 @@ import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.oracle.TiDBCERTOracle; +import sqlancer.tidb.oracle.TiDBDQPOracle; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @@ -71,6 +72,12 @@ public TestOracle create(TiDBGlobalState globalState) throws SQ public boolean requiresAllTablesToContainRows() { return true; } + }, + DQP { + @Override + public TestOracle create(TiDBGlobalState globalState) throws SQLException { + return new TiDBDQPOracle(globalState); + } }; } diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 6a95fb9a9..535cf973f 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -32,6 +32,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("can't drop"); errors.add("A PRIMARY must include all columns in the table's partitioning function"); errors.add("key was too long"); + errors.add("Duplicate entry"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); diff --git a/src/sqlancer/tidb/gen/TiDBHintGenerator.java b/src/sqlancer/tidb/gen/TiDBHintGenerator.java index b1a343c67..f58695bb8 100644 --- a/src/sqlancer/tidb/gen/TiDBHintGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBHintGenerator.java @@ -1,5 +1,7 @@ package sqlancer.tidb.gen; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -22,12 +24,13 @@ enum IndexHint { INL_HASH_JOIN, // INL_MERGE_JOIN, // HASH_JOIN, // + READ_FROM_TIKV, // + READ_FROM_TIFLASH, // HASH_AGG, // STREAM_AGG, // USE_INDEX, // IGNORE_INDEX, // AGG_TO_COP, // - // READ_FROM_STORAGE USE_INDEX_MERGE, // NO_INDEX_MERGE, // USE_TOJA, // @@ -46,13 +49,40 @@ public TiDBHintGenerator(TiDBSelect select, List tables) { } public static void generateHints(TiDBSelect select, List tables) { - new TiDBHintGenerator(select, tables).generate(); + new TiDBHintGenerator(select, tables).randomHint(); + } + public static List generateAllHints(TiDBSelect select, List tables) { + TiDBHintGenerator generator = new TiDBHintGenerator(select, tables); + return generator.allHints(); } - private void generate() { + private void randomHint() { TiDBTable table = Randomly.fromList(tables); - switch (Randomly.fromOptions(IndexHint.values())) { + IndexHint chosenhint = Randomly.fromOptions(IndexHint.values()); + generate(table, chosenhint); + } + + private List allHints() { + List results = new ArrayList<>(); + IndexHint[] values = IndexHint.values(); + List availableHints = new ArrayList<>(Arrays.asList(values)); + + for (IndexHint hint : availableHints) { + try { + TiDBText generatedHint = generate(Randomly.fromList(tables), hint); + results.add(generatedHint); + } catch (IgnoreMeException e) { + continue; + } + } + return results; + } + + private TiDBText generate(TiDBTable table, IndexHint chosenhint) { + sb.setLength(0); + + switch (chosenhint) { case MERGE_JOIN: tablesHint("MERGE_JOIN"); break; @@ -68,6 +98,12 @@ private void generate() { case HASH_JOIN: tablesHint("HASH_JOIN"); break; + case READ_FROM_TIKV: + storageHint("READ_FROM_STORAGE(TIKV"); + break; + case READ_FROM_TIFLASH: + storageHint("READ_FROM_STORAGE(TIFLASH"); + break; case HASH_AGG: sb.append("HASH_AGG()"); break; @@ -126,7 +162,9 @@ private void generate() { default: throw new AssertionError(); } - select.setHint(new TiDBText(sb.toString())); + TiDBText hint = new TiDBText(sb.toString()); + select.setHint(hint); + return hint; } private void indexesHint(String string) { @@ -152,6 +190,13 @@ private void tablesHint(String string) { sb.append(")"); } + private void storageHint(String string) { + sb.append(string); + sb.append("["); + appendTables(); + sb.append("])"); + } + private void twoTablesHint(String string, TiDBTable table) { if (table.hasIndexes()) { sb.append(string); diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index b48ea31cc..11f599fc0 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -43,7 +43,9 @@ private enum Action { TIDB_ENABLE_WINDOW_FUNCTION("tidb_enable_window_function", (r) -> Randomly.fromOptions(0, 1)), - TIDB_ENABLE_FAST_ANALYZE("tidb_enable_fast_analyze", (r) -> Randomly.fromOptions(0, 1)), // + // TIDB_ENABLE_FAST_ANALYZE("tidb_enable_fast_analyze", (r) -> Randomly.fromOptions(0, 1)), // + // java.sql.SQLException: Fast analyze hasn't reached General Availability and only support analyze version 1 + // currently TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions(0, 1)), TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // diff --git a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java new file mode 100644 index 000000000..e35e4ee70 --- /dev/null +++ b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java @@ -0,0 +1,80 @@ +package sqlancer.tidb.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.tidb.TiDBErrors; +import sqlancer.tidb.TiDBExpressionGenerator; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.TiDBSchema.TiDBTables; +import sqlancer.tidb.ast.TiDBColumnReference; +import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableReference; +import sqlancer.tidb.ast.TiDBText; +import sqlancer.tidb.gen.TiDBHintGenerator; +import sqlancer.tidb.visitor.TiDBVisitor; + +public class TiDBDQPOracle implements TestOracle { + private TiDBExpressionGenerator gen; + private final TiDBGlobalState state; + private TiDBSelect select; + private final ExpectedErrors errors = new ExpectedErrors(); + + public TiDBDQPOracle(TiDBGlobalState globalState) { + state = globalState; + TiDBErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws SQLException { + // Randomly generate a query + TiDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new TiDBExpressionGenerator(state).setColumns(tables.getColumns()); + select = new TiDBSelect(); + + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new TiDBColumnReference(c)) + .collect(Collectors.toList())); + select.setFetchColumns(fetchColumns); + + List tableList = tables.getTables().stream().map(t -> new TiDBTableReference(t)) + .collect(Collectors.toList()); + List joins = TiDBJoin.getJoins(tableList, state); + select.setJoinList(joins); + select.setFromList(tableList); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByExpressions(gen.generateOrderBys()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setOffsetClause(gen.generateExpression()); + } + + String originalQueryString = TiDBVisitor.asString(select); + List originalResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + List hintList = TiDBHintGenerator.generateAllHints(select, tables.getTables()); + for (TiDBText hint : hintList) { + select.setHint(hint); + String queryString = TiDBVisitor.asString(select); + List result = ComparatorHelper.getResultSetFirstColumnAsString(queryString, errors, state); + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, List.of(queryString), + state); + } + } + +} diff --git a/test/sqlancer/dqp/tidb/TestTiDBDQP.java b/test/sqlancer/dqp/tidb/TestTiDBDQP.java new file mode 100644 index 000000000..300dfde34 --- /dev/null +++ b/test/sqlancer/dqp/tidb/TestTiDBDQP.java @@ -0,0 +1,22 @@ +package sqlancer.dqp.tidb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestTiDBDQP { + + @Test + public void testTiDBQPG() { + String tiDB = System.getenv("TIDB_AVAILABLE"); + boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); + assumeTrue(tiDBIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "tidb", "--oracle", "DQP" })); + } + +} From 93626814cb51f77f6e86cfa95e5bced81dc0cb0f Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Mon, 18 Mar 2024 23:38:26 +0800 Subject: [PATCH 405/856] MySQL Join implementation (#921) --- .../mysql/MySQLExpectedValueVisitor.java | 13 +++ src/sqlancer/mysql/MySQLProvider.java | 10 +-- src/sqlancer/mysql/MySQLToStringVisitor.java | 39 +++++++++ src/sqlancer/mysql/MySQLVisitor.java | 10 +++ src/sqlancer/mysql/ast/MySQLJoin.java | 81 ++++++++++++++++++- src/sqlancer/mysql/ast/MySQLText.java | 14 ++++ 6 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 src/sqlancer/mysql/ast/MySQLText.java diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index fd77a9f85..0ff8e389d 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -13,10 +13,12 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; public class MySQLExpectedValueVisitor implements MySQLVisitor { @@ -153,4 +155,15 @@ public void visit(MySQLCollate collate) { visit(collate.getExpectedValue()); } + @Override + public void visit(MySQLJoin join) { + print(join); + visit(join.getOnClause()); + } + + @Override + public void visit(MySQLText text) { + print(text); + } + } diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 9392550cc..287f20a7e 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -63,11 +63,6 @@ enum Action implements AbstractAction { SELECT_INFO((g) -> new SQLQueryAdapter( "select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" + g.getDatabaseName() + "'")), // - CREATE_TABLE((g) -> { - // TODO refactor - String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); - return MySQLTableGenerator.generate(g, tableName); - }), // UPDATE(MySQLUpdateGenerator::create), // DELETE(MySQLDeleteGenerator::delete), // DROP_INDEX(MySQLDropIndex::generate); @@ -94,9 +89,6 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { case SHOW_TABLES: nrPerformed = r.getInteger(0, 1); break; - case CREATE_TABLE: - nrPerformed = r.getInteger(0, 1); - break; case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; @@ -148,7 +140,7 @@ private static int mapActions(MySQLGlobalState globalState, Action a) { @Override public void generateDatabase(MySQLGlobalState globalState) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.getNotCachedInteger(1, 2)) { String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); SQLQueryAdapter createTable = MySQLTableGenerator.generate(globalState, tableName); globalState.executeStatement(createTable); diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 06f13b008..b61e501e5 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -17,11 +17,13 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; public class MySQLToStringVisitor extends ToStringVisitor implements MySQLVisitor { @@ -278,4 +280,41 @@ public void visit(MySQLCollate collate) { sb.append(")"); } + @Override + public void visit(MySQLJoin join) { + sb.append(" "); + switch (join.getType()) { + case NATURAL: + sb.append("NATURAL "); + break; + case INNER: + sb.append("INNER "); + break; + case STRAIGHT: + sb.append("STRAIGHT_"); + break; + case LEFT: + sb.append("LEFT "); + break; + case RIGHT: + sb.append("RIGHT "); + break; + case CROSS: + sb.append("CROSS "); + break; + default: + throw new AssertionError(join.getType()); + } + sb.append("JOIN "); + sb.append(join.getTable().getName()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + @Override + public void visit(MySQLText text) { + sb.append(text.getText()); + } } diff --git a/src/sqlancer/mysql/MySQLVisitor.java b/src/sqlancer/mysql/MySQLVisitor.java index eeda4f681..7b8944f28 100644 --- a/src/sqlancer/mysql/MySQLVisitor.java +++ b/src/sqlancer/mysql/MySQLVisitor.java @@ -12,10 +12,12 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; public interface MySQLVisitor { @@ -52,6 +54,10 @@ public interface MySQLVisitor { void visit(MySQLCollate collate); + void visit(MySQLJoin join); + + void visit(MySQLText text); + default void visit(MySQLExpression expr) { if (expr instanceof MySQLConstant) { visit((MySQLConstant) expr); @@ -77,6 +83,8 @@ default void visit(MySQLExpression expr) { visit((MySQLOrderByTerm) expr); } else if (expr instanceof MySQLExists) { visit((MySQLExists) expr); + } else if (expr instanceof MySQLJoin) { + visit((MySQLJoin) expr); } else if (expr instanceof MySQLStringExpression) { visit((MySQLStringExpression) expr); } else if (expr instanceof MySQLBetweenOperation) { @@ -85,6 +93,8 @@ default void visit(MySQLExpression expr) { visit((MySQLTableReference) expr); } else if (expr instanceof MySQLCollate) { visit((MySQLCollate) expr); + } else if (expr instanceof MySQLText) { + visit((MySQLText) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/mysql/ast/MySQLJoin.java b/src/sqlancer/mysql/ast/MySQLJoin.java index 8558e43a9..13c21b7a0 100644 --- a/src/sqlancer/mysql/ast/MySQLJoin.java +++ b/src/sqlancer/mysql/ast/MySQLJoin.java @@ -1,10 +1,85 @@ package sqlancer.mysql.ast; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + public class MySQLJoin implements MySQLExpression { - @Override - public MySQLConstant getExpectedValue() { - throw new UnsupportedOperationException(); + public enum JoinType { + NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; + } + + private final MySQLTable table; + private MySQLExpression onClause; + private JoinType type; + + public MySQLJoin(MySQLJoin other) { + this.table = other.table; + this.onClause = other.onClause; + this.type = other.type; } + public MySQLJoin(MySQLTable table, MySQLExpression onClause, JoinType type) { + this.table = table; + this.onClause = onClause; + this.type = type; + } + + public MySQLTable getTable() { + return table; + } + + public MySQLExpression getOnClause() { + return onClause; + } + + public JoinType getType() { + return type; + } + + public void setOnClause(MySQLExpression onClause) { + this.onClause = onClause; + } + + public void setType(JoinType type) { + this.type = type; + } + + public static List getRandomJoinClauses(List tables, MySQLGlobalState globalState) { + List joinStatements = new ArrayList<>(); + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + List columns = new ArrayList<>(); + if (tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + MySQLTable table = Randomly.fromList(tables); + tables.remove(table); + columns.addAll(table.getColumns()); + MySQLExpressionGenerator joinGen = new MySQLExpressionGenerator(globalState).setColumns(columns); + MySQLExpression joinClause = joinGen.generateExpression(); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + joinClause = null; + } + MySQLJoin j = new MySQLJoin(table, joinClause, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } } diff --git a/src/sqlancer/mysql/ast/MySQLText.java b/src/sqlancer/mysql/ast/MySQLText.java new file mode 100644 index 000000000..36040a383 --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLText.java @@ -0,0 +1,14 @@ +package sqlancer.mysql.ast; + +public class MySQLText implements MySQLExpression { + + private final String text; + + public MySQLText(String text) { + this.text = text; + } + + public String getText() { + return text; + } +} From a546635d884ab88fba62703cfa9896f6e3db3b65 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 19 Mar 2024 19:31:01 +0800 Subject: [PATCH 406/856] MySQL DQP (#923) --- src/sqlancer/mysql/MySQLBugs.java | 9 + src/sqlancer/mysql/MySQLOptions.java | 7 + src/sqlancer/mysql/MySQLToStringVisitor.java | 5 + src/sqlancer/mysql/ast/MySQLSelect.java | 9 + .../mysql/gen/MySQLHintGenerator.java | 204 ++++++++++++++++++ src/sqlancer/mysql/gen/MySQLSetGenerator.java | 57 ++++- src/sqlancer/mysql/oracle/MySQLDQPOracle.java | 104 +++++++++ test/sqlancer/dqp/mysql/TestMySQLDQP.java | 22 ++ 8 files changed, 412 insertions(+), 5 deletions(-) create mode 100644 src/sqlancer/mysql/gen/MySQLHintGenerator.java create mode 100644 src/sqlancer/mysql/oracle/MySQLDQPOracle.java create mode 100644 test/sqlancer/dqp/mysql/TestMySQLDQP.java diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index ea44c83d4..00a7a0ddf 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -22,6 +22,15 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=111471 public static boolean bug111471 = true; + // https://bugs.mysql.com/bug.php?id=112242 + public static boolean bug112242 = true; + + // https://bugs.mysql.com/bug.php?id=112243 + public static boolean bug112243 = true; + + // https://bugs.mysql.com/bug.php?id=112264 + public static boolean bug112264 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 4e42e8b00..87408c39f 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -12,6 +12,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; import sqlancer.mysql.oracle.MySQLCERTOracle; +import sqlancer.mysql.oracle.MySQLDQPOracle; import sqlancer.mysql.oracle.MySQLFuzzer; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @@ -65,6 +66,12 @@ public TestOracle create(MySQLGlobalState globalState) throws return new MySQLFuzzer(globalState); } + }, + DQP { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLDQPOracle(globalState); + } }; } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index b61e501e5..757232406 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -38,6 +38,11 @@ public void visitSpecific(MySQLExpression expr) { @Override public void visit(MySQLSelect s) { sb.append("SELECT "); + if (s.getHint() != null) { + sb.append("/*+ "); + visit(s.getHint()); + sb.append("*/ "); + } switch (s.getFromOptions()) { case DISTINCT: sb.append("DISTINCT "); diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index 7abd1f639..a50007417 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -9,6 +9,7 @@ public class MySQLSelect extends SelectBase implements MySQLExp private SelectType fromOptions = SelectType.ALL; private List modifiers = Collections.emptyList(); + private MySQLText hint; public enum SelectType { DISTINCT, ALL, DISTINCTROW; @@ -39,4 +40,12 @@ public MySQLConstant getExpectedValue() { return null; } + public void setHint(MySQLText hint) { + this.hint = hint; + } + + public MySQLText getHint() { + return hint; + } + } diff --git a/src/sqlancer/mysql/gen/MySQLHintGenerator.java b/src/sqlancer/mysql/gen/MySQLHintGenerator.java new file mode 100644 index 000000000..141aea279 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLHintGenerator.java @@ -0,0 +1,204 @@ +package sqlancer.mysql.gen; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.mysql.MySQLSchema.MySQLIndex; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLText; + +public class MySQLHintGenerator { + + private final MySQLSelect select; + private final List tables; + private final StringBuilder sb = new StringBuilder(); + + enum OptimizeHint { + BKA, NO_BKA, BNL, NO_BNL, DERIVED_CONDITION_PUSHDOWN, NO_DERIVED_CONDITION_PUSHDOWN, GROUP_INDEX, + NO_GROUP_INDEX, HASH_JOIN, NO_HASH_JOIN, INDEX, NO_INDEX, INDEX_MERGE, NO_INDEX_MERGE, JOIN_FIXED_ORDER, + JOIN_INDEX, NO_JOIN_INDEX, JOIN_ORDER, JOIN_PREFIX, JOIN_SUFFIX, MERGE, NO_MERGE, MRR, NO_MRR, NO_ICP, + NO_RANGE_OPTIMIZATION, ORDER_INDEX, NO_ORDER_INDEX, SEMIJOIN, NO_SEMIJOIN, SKIP_SCAN, NO_SKIP_SCAN + } + + public MySQLHintGenerator(MySQLSelect select, List tables) { + this.select = select; + this.tables = tables; + } + + public static void generateHints(MySQLSelect select, List tables) { + new MySQLHintGenerator(select, tables).randomHint(); + } + + public static List generateAllHints(MySQLSelect select, List tables) { + MySQLHintGenerator generator = new MySQLHintGenerator(select, tables); + return generator.allHints(); + } + + private void randomHint() { + OptimizeHint chosenhint = Randomly.fromOptions(OptimizeHint.values()); + generate(chosenhint); + } + + private List allHints() { + List results = new ArrayList<>(); + for (OptimizeHint hint : OptimizeHint.values()) { + try { + MySQLText generatedHint = generate(hint); + results.add(generatedHint); + } catch (IgnoreMeException e) { + continue; + } + } + return results; + } + + private MySQLText generate(OptimizeHint chosenhint) { + sb.setLength(0); + + switch (chosenhint) { + case BKA: + tablesHint("BKA"); + break; + case NO_BKA: + tablesHint("NO_BKA"); + break; + case BNL: + tablesHint("BNL"); + break; + case NO_BNL: + tablesHint("NO_BNL"); + break; + case DERIVED_CONDITION_PUSHDOWN: + tablesHint("DERIVED_CONDITION_PUSHDOWN"); + break; + case NO_DERIVED_CONDITION_PUSHDOWN: + tablesHint("NO_DERIVED_CONDITION_PUSHDOWN"); + break; + case GROUP_INDEX: + indexesHint("GROUP_INDEX"); + break; + case NO_GROUP_INDEX: + indexesHint("NO_GROUP_INDEX"); + break; + case HASH_JOIN: + tablesHint("HASH_JOIN"); + break; + case NO_HASH_JOIN: + tablesHint("NO_HASH_JOIN"); + break; + case INDEX: + indexesHint("INDEX"); + break; + case NO_INDEX: + indexesHint("NO_INDEX"); + break; + case INDEX_MERGE: + indexesHint("INDEX_MERGE"); + break; + case NO_INDEX_MERGE: + indexesHint("NO_INDEX_MERGE"); + break; + case JOIN_FIXED_ORDER: + tablesHint("JOIN_FIXED_ORDER"); + break; + case JOIN_INDEX: + indexesHint("JOIN_INDEX"); + break; + case NO_JOIN_INDEX: + indexesHint("NO_JOIN_INDEX"); + break; + case JOIN_ORDER: + tablesHint("JOIN_ORDER"); + break; + case JOIN_PREFIX: + tablesHint("JOIN_PREFIX"); + break; + case JOIN_SUFFIX: + tablesHint("JOIN_SUFFIX"); + break; + case MERGE: + tablesHint("MERGE"); + break; + case NO_MERGE: + tablesHint("NO_MERGE"); + break; + case MRR: + indexesHint("MRR"); + break; + case NO_MRR: + indexesHint("NO_MRR"); + break; + case NO_ICP: + indexesHint("NO_ICP"); + break; + case NO_RANGE_OPTIMIZATION: + indexesHint("NO_RANGE_OPTIMIZATION"); + break; + case ORDER_INDEX: + indexesHint("ORDER_INDEX"); + break; + case NO_ORDER_INDEX: + indexesHint("NO_ORDER_INDEX"); + break; + case SEMIJOIN: + semiHint("SEMIJOIN"); + break; + case NO_SEMIJOIN: + semiHint("NO_SEMIJOIN"); + break; + case SKIP_SCAN: + indexesHint("SKIP_SCAN"); + break; + case NO_SKIP_SCAN: + indexesHint("NO_SKIP_SCAN"); + break; + default: + throw new AssertionError(); + } + MySQLText hint = new MySQLText(sb.toString()); + select.setHint(hint); + return hint; + } + + private void indexesHint(String string) { + sb.append(string); + sb.append("("); + MySQLTable table = Randomly.fromList(tables); + List allIndexes = table.getIndexes(); + sb.append(table.getName()); + sb.append(", "); + if (allIndexes.isEmpty()) { + sb.append("PRIMARY"); + } else { + List indexSubset = Randomly.nonEmptySubset(allIndexes); + sb.append(indexSubset.stream().map(i -> i.getIndexName()).distinct().collect(Collectors.joining(", "))); + } + sb.append(")"); + } + + private void tablesHint(String string) { + sb.append(string); + sb.append("("); + appendTables(); + sb.append(")"); + } + + private void semiHint(String string) { + sb.append(string); + sb.append("("); + String[] options = { "DUPSWEEDOUT", "FIRSTMATCH", "LOOSESCAN", "MATERIALIZATION" }; + List chosenOptions = Randomly.nonEmptySubset(options); + sb.append(chosenOptions.stream().collect(Collectors.joining(", "))); + sb.append(")"); + } + + private void appendTables() { + List tableSubset = Randomly.nonEmptySubset(tables); + sb.append(tableSubset.stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + } + +} diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 821184617..94db1fab7 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -1,5 +1,7 @@ package sqlancer.mysql.gen; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; @@ -7,6 +9,7 @@ import sqlancer.MainOptions; import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; public class MySQLSetGenerator { @@ -131,11 +134,13 @@ private enum Action { private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); - String[] options = { "batched_key_access", "block_nested_loop", "condition_fanout_filter", "derived_merge", - "engine_condition_pushdown", "index_condition_pushdown", "use_index_extensions", "index_merge", - "index_merge_intersection", "index_merge_sort_union", "index_merge_union", "use_invisible_indexes", - "mrr", "mrr_cost_based", "skip_scan", "semijoin", "duplicateweedout", "firstmatch", "loosescan", - "materialization", "subquery_materialization_cost_based" }; + String[] options = { "index_merge", "index_merge_union", "index_merge_sort_union", + "index_merge_intersection", "engine_condition_pushdown", "index_condition_pushdown", "mrr", + "mrr_cost_based", "block_nested_loop", "batched_key_access", "materialization", "semijoin", + "loosescan", "firstmatch", "duplicateweedout", "subquery_materialization_cost_based", + "use_index_extensions", "condition_fanout_filter", "derived_merge", "use_invisible_indexes", + "skip_scan", "hash_join", "subquery_to_derived", "prefer_ordering_index", + "derived_condition_pushdown" }; List optionSubset = Randomly.nonEmptySubset(options); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); @@ -188,4 +193,46 @@ private SQLQueryAdapter get() { return new SQLQueryAdapter(sb.toString()); } + public static SQLQueryAdapter resetOptimizer() { + return new SQLQueryAdapter("SET optimizer_switch='default'"); + } + + public static List getAllOptimizer(MySQLGlobalState globalState) { + List result = new ArrayList<>(); + String[] options = { "index_merge", "index_merge_union", "index_merge_sort_union", "index_merge_intersection", + "engine_condition_pushdown", "index_condition_pushdown", "mrr", "mrr_cost_based", "block_nested_loop", + "batched_key_access", "materialization", "semijoin", "loosescan", "firstmatch", "duplicateweedout", + "subquery_materialization_cost_based", "use_index_extensions", "condition_fanout_filter", + "derived_merge", "use_invisible_indexes", "skip_scan", "hash_join", "subquery_to_derived", + "prefer_ordering_index", "derived_condition_pushdown" }; + + List availableOptions = new ArrayList<>(Arrays.asList(options)); + if (MySQLBugs.bug112242) { + availableOptions.remove("use_invisible_indexes"); + } + if (MySQLBugs.bug112243) { + availableOptions.remove("subquery_to_derived"); + } + if (MySQLBugs.bug112264) { + availableOptions.remove("block_nested_loop"); + } + + StringBuilder sb = new StringBuilder(); + sb.append("SET "); + if (globalState.getOptions().getNumberConcurrentThreads() == 1 && Randomly.getBoolean()) { + sb.append("GLOBAL"); + } else { + sb.append("SESSION"); + } + sb.append(" optimizer_switch = '%s'"); + + for (String option : availableOptions) { + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=on"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=off"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=default"))); + } + + return result; + } + } diff --git a/src/sqlancer/mysql/oracle/MySQLDQPOracle.java b/src/sqlancer/mysql/oracle/MySQLDQPOracle.java new file mode 100644 index 000000000..414ffb156 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLDQPOracle.java @@ -0,0 +1,104 @@ +package sqlancer.mysql.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLJoin; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLText; +import sqlancer.mysql.gen.MySQLExpressionGenerator; +import sqlancer.mysql.gen.MySQLHintGenerator; +import sqlancer.mysql.gen.MySQLSetGenerator; + +public class MySQLDQPOracle implements TestOracle { + private final MySQLGlobalState state; + private MySQLExpressionGenerator gen; + private MySQLSelect select; + private final ExpectedErrors errors = new ExpectedErrors(); + + public MySQLDQPOracle(MySQLGlobalState globalState) { + state = globalState; + MySQLErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws Exception { + // Randomly generate a query + MySQLTables tables = state.getSchema().getRandomTableNonEmptyTables(); + gen = new MySQLExpressionGenerator(state).setColumns(tables.getColumns()); + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() + .map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList())); + + select = new MySQLSelect(); + select.setFetchColumns(fetchColumns); + + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByExpressions(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateExpression()); + } + } + + // Set the join. + List joinExpressions = MySQLJoin.getRandomJoinClauses(tables.getTables(), state); + select.setJoinList(joinExpressions.stream().map(j -> (MySQLExpression) j).collect(Collectors.toList())); + + // Set the from clause from the tables that are not used in the join. + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) + .collect(Collectors.toList()); + select.setFromList(tableList); + + // Get the result of the first query + String originalQueryString = MySQLVisitor.asString(select); + List originalResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + // Check hints + List hintList = MySQLHintGenerator.generateAllHints(select, tables.getTables()); + for (MySQLText hint : hintList) { + select.setHint(hint); + String queryString = MySQLVisitor.asString(select); + List result = ComparatorHelper.getResultSetFirstColumnAsString(queryString, errors, state); + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, List.of(queryString), + state); + } + + // Check optimizer variables + List optimizationList = MySQLSetGenerator.getAllOptimizer(state); + for (SQLQueryAdapter optimization : optimizationList) { + optimization.execute(state); + List result = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, + List.of(originalQueryString), state); + } catch (AssertionError e) { + String assertionMessage = String.format( + "The size of the result sets mismatch (%d and %d)!" + System.lineSeparator() + + "First query: \"%s\", whose cardinality is: %d" + System.lineSeparator() + + "Second query:\"%s\", whose cardinality is: %d", + originalResult.size(), result.size(), originalQueryString, originalResult.size(), + String.join(";", originalQueryString), result.size()); + assertionMessage += System.lineSeparator() + "The setting: " + optimization.getQueryString(); + throw new AssertionError(assertionMessage); + } + } + } +} diff --git a/test/sqlancer/dqp/mysql/TestMySQLDQP.java b/test/sqlancer/dqp/mysql/TestMySQLDQP.java new file mode 100644 index 000000000..38e5eef61 --- /dev/null +++ b/test/sqlancer/dqp/mysql/TestMySQLDQP.java @@ -0,0 +1,22 @@ +package sqlancer.dqp.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestMySQLDQP { + + @Test + public void testmysqlQPG() { + String mysql = System.getenv("MYSQL_AVAILABLE"); + boolean mysqlIsAvailable = mysql != null && mysql.equalsIgnoreCase("true"); + assumeTrue(mysqlIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "mysql", "--oracle", "DQP" })); + } + +} From afc6792f1324b0675e3fe8bc879adc35e793fe26 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 19 Mar 2024 20:23:28 +0800 Subject: [PATCH 407/856] MariaDB Join implementation (#924) --- src/sqlancer/mariadb/MariaDBSchema.java | 4 + .../mariadb/ast/MariaDBAggregate.java | 2 +- .../mariadb/ast/MariaDBBinaryOperator.java | 2 +- .../mariadb/ast/MariaDBColumnName.java | 2 +- src/sqlancer/mariadb/ast/MariaDBConstant.java | 5 +- .../mariadb/ast/MariaDBExpression.java | 2 +- src/sqlancer/mariadb/ast/MariaDBFunction.java | 2 +- .../mariadb/ast/MariaDBInOperation.java | 2 +- src/sqlancer/mariadb/ast/MariaDBJoin.java | 86 +++++++++++++++++++ .../ast/MariaDBPostfixUnaryOperation.java | 2 +- .../mariadb/ast/MariaDBSelectStatement.java | 7 +- .../mariadb/ast/MariaDBStringVisitor.java | 40 ++++++++- src/sqlancer/mariadb/ast/MariaDBText.java | 2 +- .../ast/MariaDBUnaryPrefixOperation.java | 2 +- src/sqlancer/mariadb/ast/MariaDBVisitor.java | 4 + 15 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 src/sqlancer/mariadb/ast/MariaDBJoin.java diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index ed432e8da..d1bcfbd97 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -242,4 +242,8 @@ public MariaDBSchema(List databaseTables) { super(databaseTables); } + public MariaDBTables getRandomTableNonEmptyTables() { + return new MariaDBTables(Randomly.nonEmptySubset(getDatabaseTables())); + } + } diff --git a/src/sqlancer/mariadb/ast/MariaDBAggregate.java b/src/sqlancer/mariadb/ast/MariaDBAggregate.java index 3977a6a95..ab5a47781 100644 --- a/src/sqlancer/mariadb/ast/MariaDBAggregate.java +++ b/src/sqlancer/mariadb/ast/MariaDBAggregate.java @@ -1,6 +1,6 @@ package sqlancer.mariadb.ast; -public class MariaDBAggregate extends MariaDBExpression { +public class MariaDBAggregate implements MariaDBExpression { private final MariaDBExpression expr; private final MariaDBAggregateFunction aggr; diff --git a/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java b/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java index 14f3c308a..5f56d168f 100644 --- a/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java +++ b/src/sqlancer/mariadb/ast/MariaDBBinaryOperator.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class MariaDBBinaryOperator extends MariaDBExpression { +public class MariaDBBinaryOperator implements MariaDBExpression { private MariaDBExpression left; private MariaDBExpression right; diff --git a/src/sqlancer/mariadb/ast/MariaDBColumnName.java b/src/sqlancer/mariadb/ast/MariaDBColumnName.java index a4fd78c1b..65d4cea0c 100644 --- a/src/sqlancer/mariadb/ast/MariaDBColumnName.java +++ b/src/sqlancer/mariadb/ast/MariaDBColumnName.java @@ -2,7 +2,7 @@ import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; -public class MariaDBColumnName extends MariaDBExpression { +public class MariaDBColumnName implements MariaDBExpression { private final MariaDBColumn column; diff --git a/src/sqlancer/mariadb/ast/MariaDBConstant.java b/src/sqlancer/mariadb/ast/MariaDBConstant.java index 6e3d77691..8670c9ed2 100644 --- a/src/sqlancer/mariadb/ast/MariaDBConstant.java +++ b/src/sqlancer/mariadb/ast/MariaDBConstant.java @@ -1,6 +1,9 @@ package sqlancer.mariadb.ast; -public class MariaDBConstant extends MariaDBExpression { +public class MariaDBConstant implements MariaDBExpression { + + private MariaDBConstant() { + } public static class MariaDBNullConstant extends MariaDBConstant { diff --git a/src/sqlancer/mariadb/ast/MariaDBExpression.java b/src/sqlancer/mariadb/ast/MariaDBExpression.java index beff58866..5caec1b40 100644 --- a/src/sqlancer/mariadb/ast/MariaDBExpression.java +++ b/src/sqlancer/mariadb/ast/MariaDBExpression.java @@ -1,5 +1,5 @@ package sqlancer.mariadb.ast; -public class MariaDBExpression { +public interface MariaDBExpression { } diff --git a/src/sqlancer/mariadb/ast/MariaDBFunction.java b/src/sqlancer/mariadb/ast/MariaDBFunction.java index fd033d310..481a26aaf 100644 --- a/src/sqlancer/mariadb/ast/MariaDBFunction.java +++ b/src/sqlancer/mariadb/ast/MariaDBFunction.java @@ -2,7 +2,7 @@ import java.util.List; -public class MariaDBFunction extends MariaDBExpression { +public class MariaDBFunction implements MariaDBExpression { private final MariaDBFunctionName func; private final List args; diff --git a/src/sqlancer/mariadb/ast/MariaDBInOperation.java b/src/sqlancer/mariadb/ast/MariaDBInOperation.java index 85981b390..15aad4daa 100644 --- a/src/sqlancer/mariadb/ast/MariaDBInOperation.java +++ b/src/sqlancer/mariadb/ast/MariaDBInOperation.java @@ -2,7 +2,7 @@ import java.util.List; -public class MariaDBInOperation extends MariaDBExpression { +public class MariaDBInOperation implements MariaDBExpression { private final MariaDBExpression expr; private final List list; diff --git a/src/sqlancer/mariadb/ast/MariaDBJoin.java b/src/sqlancer/mariadb/ast/MariaDBJoin.java new file mode 100644 index 000000000..b8110b7f2 --- /dev/null +++ b/src/sqlancer/mariadb/ast/MariaDBJoin.java @@ -0,0 +1,86 @@ +package sqlancer.mariadb.ast; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; +import sqlancer.mariadb.gen.MariaDBExpressionGenerator; + +public class MariaDBJoin implements MariaDBExpression { + + public enum JoinType { + NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; + } + + private final MariaDBTable table; + private MariaDBExpression onClause; + private JoinType type; + + public MariaDBJoin(MariaDBJoin other) { + this.table = other.table; + this.onClause = other.onClause; + this.type = other.type; + } + + public MariaDBJoin(MariaDBTable table, MariaDBExpression onClause, JoinType type) { + this.table = table; + this.onClause = onClause; + this.type = type; + } + + public MariaDBTable getTable() { + return table; + } + + public MariaDBExpression getOnClause() { + return onClause; + } + + public JoinType getType() { + return type; + } + + public void setOnClause(MariaDBExpression onClause) { + this.onClause = onClause; + } + + public void setType(JoinType type) { + this.type = type; + } + + public static List getRandomJoinClauses(List tables, MariaDBGlobalState globalState) { + List joinStatements = new ArrayList<>(); + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + List columns = new ArrayList<>(); + if (tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + MariaDBTable table = Randomly.fromList(tables); + tables.remove(table); + columns.addAll(table.getColumns()); + MariaDBExpressionGenerator joinGen = new MariaDBExpressionGenerator(globalState.getRandomly()) + .setColumns(columns); + MariaDBExpression joinClause = joinGen.getRandomExpression(); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + joinClause = null; + } + MariaDBJoin j = new MariaDBJoin(table, joinClause, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } +} diff --git a/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java b/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java index cd655a5d1..9b56a44dd 100644 --- a/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java +++ b/src/sqlancer/mariadb/ast/MariaDBPostfixUnaryOperation.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class MariaDBPostfixUnaryOperation extends MariaDBExpression { +public class MariaDBPostfixUnaryOperation implements MariaDBExpression { private MariaDBPostfixUnaryOperator operator; private MariaDBExpression randomWhereCondition; diff --git a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java index d1700f508..83b944a17 100644 --- a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java +++ b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java @@ -3,12 +3,13 @@ import java.util.ArrayList; import java.util.List; +import sqlancer.common.ast.SelectBase; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -public class MariaDBSelectStatement extends MariaDBExpression { +public class MariaDBSelectStatement extends SelectBase implements MariaDBExpression { public enum MariaDBSelectType { - ALL + ALL, DISTINCT, DISTINCTROW; } private List groupBys = new ArrayList<>(); @@ -21,6 +22,7 @@ public void setGroupByClause(List groupBys) { this.groupBys = groupBys; } + @Override public void setFetchColumns(List columns) { this.columns = columns; @@ -34,6 +36,7 @@ public void setSelectType(MariaDBSelectType selectType) { this.selectType = selectType; } + @Override public void setWhereClause(MariaDBExpression whereCondition) { this.whereCondition = whereCondition; } diff --git a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java index 53ed490d0..07951d9ac 100644 --- a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java @@ -27,7 +27,7 @@ public void visit(MariaDBPostfixUnaryOperation op) { @Override public void visit(MariaDBColumnName c) { - sb.append(c.getColumn().getName()); + sb.append(c.getColumn().getFullQualifiedName()); } @Override @@ -42,12 +42,16 @@ public void visit(MariaDBSelectStatement s) { } sb.append(" FROM "); sb.append(s.getTables().stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + + for (MariaDBExpression j : s.getJoinList()) { + visit(j); + } if (s.getWhereCondition() != null) { sb.append(" WHERE "); visit(s.getWhereCondition()); } if (s.getGroupBys().size() != 0) { - sb.append(" GROUP BY"); + sb.append(" GROUP BY "); for (i = 0; i < s.getGroupBys().size(); i++) { if (i != 0) { sb.append(", "); @@ -131,4 +135,36 @@ private void visitList(List list) { } } + @Override + public void visit(MariaDBJoin join) { + sb.append(" "); + switch (join.getType()) { + case NATURAL: + sb.append("NATURAL "); + break; + case INNER: + sb.append("INNER "); + break; + case STRAIGHT: + sb.append("STRAIGHT_"); + break; + case LEFT: + sb.append("LEFT "); + break; + case RIGHT: + sb.append("RIGHT "); + break; + case CROSS: + sb.append("CROSS "); + break; + default: + throw new AssertionError(join.getType()); + } + sb.append("JOIN "); + sb.append(join.getTable().getName()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } } diff --git a/src/sqlancer/mariadb/ast/MariaDBText.java b/src/sqlancer/mariadb/ast/MariaDBText.java index b96871063..c3d1c2d1a 100644 --- a/src/sqlancer/mariadb/ast/MariaDBText.java +++ b/src/sqlancer/mariadb/ast/MariaDBText.java @@ -1,6 +1,6 @@ package sqlancer.mariadb.ast; -public class MariaDBText extends MariaDBExpression { +public class MariaDBText implements MariaDBExpression { private final MariaDBExpression expr; private final String text; diff --git a/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java b/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java index af229ce6e..dda61dbee 100644 --- a/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java +++ b/src/sqlancer/mariadb/ast/MariaDBUnaryPrefixOperation.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class MariaDBUnaryPrefixOperation extends MariaDBExpression { +public class MariaDBUnaryPrefixOperation implements MariaDBExpression { private MariaDBExpression expr; private MariaDBUnaryPrefixOperator op; diff --git a/src/sqlancer/mariadb/ast/MariaDBVisitor.java b/src/sqlancer/mariadb/ast/MariaDBVisitor.java index 8ea3f9d37..af94394b9 100644 --- a/src/sqlancer/mariadb/ast/MariaDBVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBVisitor.java @@ -22,6 +22,8 @@ public abstract class MariaDBVisitor { public abstract void visit(MariaDBInOperation op); + public abstract void visit(MariaDBJoin join); + public void visit(MariaDBExpression expr) { if (expr instanceof MariaDBConstant) { visit((MariaDBConstant) expr); @@ -43,6 +45,8 @@ public void visit(MariaDBExpression expr) { visit((MariaDBFunction) expr); } else if (expr instanceof MariaDBInOperation) { visit((MariaDBInOperation) expr); + } else if (expr instanceof MariaDBJoin) { + visit((MariaDBJoin) expr); } else { throw new AssertionError(expr.getClass()); } From 094756668c151084cf567e401aa8cca83370555d Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Tue, 19 Mar 2024 21:20:55 +0800 Subject: [PATCH 408/856] MariaDB DQP (#925) --- src/sqlancer/mariadb/MariaDBBugs.java | 21 +++++ src/sqlancer/mariadb/MariaDBErrors.java | 17 ++++ src/sqlancer/mariadb/MariaDBOptions.java | 7 ++ src/sqlancer/mariadb/MariaDBProvider.java | 2 +- .../mariadb/gen/MariaDBIndexGenerator.java | 1 + .../mariadb/gen/MariaDBSetGenerator.java | 92 ++++++++++++++----- .../MariaDBTableAdminCommandGenerator.java | 3 +- .../mariadb/gen/MariaDBTruncateGenerator.java | 6 +- .../mariadb/oracle/MariaDBDQPOracle.java | 91 ++++++++++++++++++ test/sqlancer/dqp/mariadb/TestMariaDBDQP.java | 22 +++++ 10 files changed, 236 insertions(+), 26 deletions(-) create mode 100644 src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java create mode 100644 test/sqlancer/dqp/mariadb/TestMariaDBDQP.java diff --git a/src/sqlancer/mariadb/MariaDBBugs.java b/src/sqlancer/mariadb/MariaDBBugs.java index 7a568b8fe..fb1770763 100644 --- a/src/sqlancer/mariadb/MariaDBBugs.java +++ b/src/sqlancer/mariadb/MariaDBBugs.java @@ -5,6 +5,27 @@ public final class MariaDBBugs { // https://jira.mariadb.org/browse/MDEV-21058 public static boolean bug21058 = true; + // https://jira.mariadb.org/browse/MDEV-32076 + public static boolean bug32076 = true; + + // https://jira.mariadb.org/browse/MDEV-32099 + public static boolean bug32099 = true; + + // https://jira.mariadb.org/browse/MDEV-32105 + public static boolean bug32105 = true; + + // https://jira.mariadb.org/browse/MDEV-32106 + public static boolean bug32106 = true; + + // https://jira.mariadb.org/browse/MDEV-32107 + public static boolean bug32107 = true; + + // https://jira.mariadb.org/browse/MDEV-32108 + public static boolean bug32108 = true; + + // https://jira.mariadb.org/browse/MDEV-32143 + public static boolean bug32143 = true; + private MariaDBBugs() { } diff --git a/src/sqlancer/mariadb/MariaDBErrors.java b/src/sqlancer/mariadb/MariaDBErrors.java index ab1bec4cd..4918accfb 100644 --- a/src/sqlancer/mariadb/MariaDBErrors.java +++ b/src/sqlancer/mariadb/MariaDBErrors.java @@ -7,6 +7,23 @@ public final class MariaDBErrors { private MariaDBErrors() { } + public static void addCommonErrors(ExpectedErrors errors) { + errors.add("is out of range"); + // regex + errors.add("unmatched parentheses"); + errors.add("nothing to repeat at offset"); + errors.add("missing )"); + errors.add("missing terminating ]"); + errors.add("range out of order in character class"); + errors.add("unrecognized character after "); + errors.add("Got error '(*VERB) not recognized or malformed"); + errors.add("must be followed by"); + errors.add("malformed number or name after"); + errors.add("digit expected after"); + errors.add("Regex error"); + errors.add("Lock wait timeout exceeded"); + } + public static void addInsertErrors(ExpectedErrors errors) { errors.add("Out of range"); errors.add("Duplicate entry"); // violates UNIQUE constraint diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index 70e0aedbd..f601ab0a6 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -12,6 +12,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.mariadb.MariaDBOptions.MariaDBOracleFactory; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.oracle.MariaDBDQPOracle; import sqlancer.mariadb.oracle.MariaDBNoRECOracle; @Parameters(separators = "=", commandDescription = "MariaDB (default port: " + MariaDBOptions.DEFAULT_PORT @@ -32,6 +33,12 @@ public TestOracle create(MariaDBGlobalState globalState) thr return new MariaDBNoRECOracle(globalState); } + }, + DQP { + @Override + public TestOracle create(MariaDBGlobalState globalState) throws SQLException { + return new MariaDBDQPOracle(globalState); + } } } diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index c84691203..48254b8ec 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -53,7 +53,7 @@ enum Action { public void generateDatabase(MariaDBGlobalState globalState) throws Exception { MainOptions options = globalState.getOptions(); - while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { + while (globalState.getSchema().getDatabaseTables().size() < Randomly.getNotCachedInteger(1, 3)) { String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); SQLQueryAdapter createTable = MariaDBTableGenerator.generate(tableName, globalState.getRandomly(), globalState.getSchema()); diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 19519ca2e..1ba3fbd4d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -19,6 +19,7 @@ public static SQLQueryAdapter generate(MariaDBSchema s) { ExpectedErrors errors = new ExpectedErrors(); StringBuilder sb = new StringBuilder("CREATE "); errors.add("Key/Index cannot be defined on a virtual generated column"); + errors.add("Specified key was too long"); if (Randomly.getBoolean()) { errors.add("Duplicate entry"); errors.add("Key/Index cannot be defined on a virtual generated column"); diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index f4caf0ec6..fc1f1d5a6 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -1,5 +1,6 @@ package sqlancer.mariadb.gen; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.function.Function; @@ -9,6 +10,8 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mariadb.MariaDBBugs; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; public class MariaDBSetGenerator { @@ -41,7 +44,6 @@ private enum Action { // Scope.SESSION), CONCURRENT_INSERT("concurrent_insert", (r) -> Randomly.fromOptions("NEVER", "AUTO", "ALWAYS", "0", "1", "2"), Scope.GLOBAL), - CTE_MAX_RECURSION_DEPTH("cte_max_recursion_depth", (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), DELAY_KEY_WRITE("delay_key_write", (r) -> Randomly.fromOptions("ON", "OFF", "ALL"), Scope.GLOBAL), EQ_RANGE_INDEX_DIVE_LIMIT("eq_range_index_dive_limit", (r) -> r.getLong(0, 4294967295L), Scope.GLOBAL), FLUSH("flush", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL), @@ -86,7 +88,6 @@ private enum Action { */ // READ_BUFFER_SIZE("read_buffer_size", (r) -> r.getLong(8200, 2147479552), Scope.GLOBAL, Scope.SESSION), // READ_RND_BUFFER_SIZE("read_rnd_buffer_size", (r) -> r.getLong(1, 2147483647), Scope.GLOBAL, Scope.SESSION), - SCHEMA_DEFINITION_CACHE("schema_definition_cache", (r) -> r.getLong(256, 524288), Scope.GLOBAL), /* * sort_buffer_size is commented out as a workaround for https://bugs.mysql.com/bug.php?id=95969 */ @@ -120,27 +121,16 @@ private enum Action { private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); - String[] options = { /* - * ("batched_key_access", /*"block_nested_loop", "condition_fanout_filter", - */ - "condition_pushdown_for_derived", // MariaDB - "derived_merge", // - "derived_with_keys", // MariaDB - "engine_condition_pushdown", // - "exists_to_in", // MariaDB - "extended_keys", // MariaDB - "firstmatch", // MariaDB - "index_condition_pushdown", // - /* "use_index_extensions", */ - "index_merge", // - "index_merge_intersection", // - "index_merge_sort_intersection", // - "index_merge_sort_union", // - "index_merge_union", "in_to_exists", // MariaDB - /* "use_invisible_indexes", */ "mrr", "mrr_cost_based", /* "skip_scan", */ "semijoin", /* - * "duplicateweedout", - */ - "firstmatch", "loosescan", "materialization", /* "subquery_materialization_cost_based" */ }; + String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "engine_condition_pushdown", + "exists_to_in", "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", + "index_merge", "index_merge_intersection", "index_merge_sort_intersection", + "index_merge_sort_union", "index_merge_union", "in_to_exists", "join_cache_bka", + "join_cache_hashed", "join_cache_incremental", "loosescan", "materialization", "mrr", + "mrr_cost_based", "mrr_sort_keys", "not_null_range_scan", "optimize_join_buffer_size", + "orderby_uses_equalities", "outer_join_with_cache", "partial_match_rowid_merge", + "partial_match_table_scan", "rowid_filter", "semijoin", "semijoin_with_cache", "split_materialized", + "subquery_cache", "table_elimination" }; List optionSubset = Arrays.asList(Randomly.fromOptions(options)); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); @@ -194,4 +184,60 @@ private SQLQueryAdapter get() { .from("At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'")); } + public static SQLQueryAdapter resetOptimizer() { + return new SQLQueryAdapter("SET optimizer_switch='default'"); + } + + public static List getAllOptimizer(MariaDBGlobalState globalState) { + List result = new ArrayList<>(); + String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "engine_condition_pushdown", + "exists_to_in", "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", + "index_merge", "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", + "index_merge_union", "in_to_exists", "join_cache_bka", "join_cache_hashed", "join_cache_incremental", + "loosescan", "materialization", "mrr", "mrr_cost_based", "mrr_sort_keys", "not_null_range_scan", + "optimize_join_buffer_size", "orderby_uses_equalities", "outer_join_with_cache", + "partial_match_rowid_merge", "partial_match_table_scan", "rowid_filter", "semijoin", + "semijoin_with_cache", "split_materialized", "subquery_cache", "table_elimination" }; + List availableOptions = new ArrayList<>(Arrays.asList(options)); + if (MariaDBBugs.bug21058) { + availableOptions.remove("in_to_exists"); // https://jira.mariadb.org/browse/MDEV-21058 + } + if (MariaDBBugs.bug32076) { + availableOptions.remove("not_null_range_scan"); // https://jira.mariadb.org/browse/MDEV-32076 + } + if (MariaDBBugs.bug32099) { + availableOptions.remove("optimize_join_buffer_size"); // https://jira.mariadb.org/browse/MDEV-32099 + } + if (MariaDBBugs.bug32105) { + availableOptions.remove("join_cache_hashed"); // https://jira.mariadb.org/browse/MDEV-32105 + } + if (MariaDBBugs.bug32106) { + availableOptions.remove("outer_join_with_cache"); // https://jira.mariadb.org/browse/MDEV-32106 + } + if (MariaDBBugs.bug32107) { + availableOptions.remove("table_elimination"); // https://jira.mariadb.org/browse/MDEV-32107 + } + if (MariaDBBugs.bug32108) { + availableOptions.remove("join_cache_incremental"); // https://jira.mariadb.org/browse/MDEV-32108 + } + if (MariaDBBugs.bug32143) { + availableOptions.remove("mrr"); // https://jira.mariadb.org/browse/MDEV-32143 + } + + StringBuilder sb = new StringBuilder(); + sb.append("SET SESSION optimizer_switch = '%s'"); + + for (String option : availableOptions) { + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=on"), ExpectedErrors.from( + "At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=off"), ExpectedErrors.from( + "At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'"))); + result.add(new SQLQueryAdapter(String.format(sb.toString(), option + "=default"), ExpectedErrors.from( + "At least one of the 'in_to_exists' or 'materialization' optimizer_switch flags must be 'on'"))); + } + + return result; + } + } diff --git a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java index bff83e1e0..a141ca58d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java @@ -57,7 +57,8 @@ public static SQLQueryAdapter optimizeTable(MariaDBSchema newSchema) { MariaDBCommon.addWaitClause(sb); return checkForMsgText(sb, s -> s.equals("OK") || s.equals("Table does not support optimize, doing recreate + analyze instead") - || s.contentEquals("Table is already up to date")); + || s.contentEquals("Table is already up to date") || s.contains("Lock wait timeout") + || s.contains("Operation failed")); } private static SQLQueryAdapter checkForMsgText(StringBuilder sb, Function checker) { diff --git a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java index 04e567444..ecc240bb3 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTruncateGenerator.java @@ -1,6 +1,8 @@ package sqlancer.mariadb.gen; +import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBSchema; public final class MariaDBTruncateGenerator { @@ -13,7 +15,9 @@ public static SQLQueryAdapter truncate(MariaDBSchema s) { sb.append(s.getRandomTable().getName()); sb.append(" "); MariaDBCommon.addWaitClause(sb); - return new SQLQueryAdapter(sb.toString()); + ExpectedErrors errors = new ExpectedErrors(); + MariaDBErrors.addCommonErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java new file mode 100644 index 000000000..a92be5d1e --- /dev/null +++ b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java @@ -0,0 +1,91 @@ +package sqlancer.mariadb.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mariadb.MariaDBErrors; +import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; +import sqlancer.mariadb.MariaDBSchema; +import sqlancer.mariadb.MariaDBSchema.MariaDBTables; +import sqlancer.mariadb.ast.MariaDBColumnName; +import sqlancer.mariadb.ast.MariaDBExpression; +import sqlancer.mariadb.ast.MariaDBJoin; +import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBVisitor; +import sqlancer.mariadb.gen.MariaDBExpressionGenerator; +import sqlancer.mariadb.gen.MariaDBSetGenerator; + +public class MariaDBDQPOracle implements TestOracle { + private final MariaDBGlobalState state; + private final MariaDBSchema s; + private MariaDBExpressionGenerator gen; + private MariaDBSelectStatement select; + private final ExpectedErrors errors = new ExpectedErrors(); + + public MariaDBDQPOracle(MariaDBGlobalState globalState) { + state = globalState; + s = globalState.getSchema(); + MariaDBErrors.addCommonErrors(errors); + } + + @Override + public void check() throws Exception { + MariaDBTables tables = s.getRandomTableNonEmptyTables(); + gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(tables.getColumns()) + .setCon(state.getConnection()).setState(state.getState()); + + List fetchColumns = new ArrayList<>(); + fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new MariaDBColumnName(c)) + .collect(Collectors.toList())); + + select = new MariaDBSelectStatement(); + select.setFetchColumns(fetchColumns); + + select.setSelectType(Randomly.fromOptions(MariaDBSelectStatement.MariaDBSelectType.values())); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.getRandomExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByClause(fetchColumns); + } + + // Set the join. + List joinExpressions = MariaDBJoin.getRandomJoinClauses(tables.getTables(), state); + select.setJoinList(joinExpressions.stream().map(j -> (MariaDBExpression) j).collect(Collectors.toList())); + + // Set the from clause from the tables that are not used in the join. + select.setFromTables(tables.getTables()); + + // Get the result of the first query + String originalQueryString = MariaDBVisitor.asString(select); + List originalResult = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + List optimizationList = MariaDBSetGenerator.getAllOptimizer(state); + for (SQLQueryAdapter optimization : optimizationList) { + optimization.execute(state); + List result = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResult, result, originalQueryString, + List.of(originalQueryString), state); + } catch (AssertionError e) { + String assertionMessage = String.format( + "The size of the result sets mismatch (%d and %d)!" + System.lineSeparator() + + "First query: \"%s\", whose cardinality is: %d" + System.lineSeparator() + + "Second query:\"%s\", whose cardinality is: %d", + originalResult.size(), result.size(), originalQueryString, originalResult.size(), + String.join(";", originalQueryString), result.size()); + assertionMessage += System.lineSeparator() + "The setting: " + optimization.getQueryString(); + throw new AssertionError(assertionMessage); + } + } + + } + +} diff --git a/test/sqlancer/dqp/mariadb/TestMariaDBDQP.java b/test/sqlancer/dqp/mariadb/TestMariaDBDQP.java new file mode 100644 index 000000000..47dabc225 --- /dev/null +++ b/test/sqlancer/dqp/mariadb/TestMariaDBDQP.java @@ -0,0 +1,22 @@ +package sqlancer.dqp.mariadb; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestMariaDBDQP { + + @Test + public void testMariaDBDQPMethod() { + String mariadb = System.getenv("MARIADB_AVAILABLE"); + boolean mariadbIsAvailable = mariadb != null && mariadb.equalsIgnoreCase("true"); + assumeTrue(mariadbIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "mariadb", "--oracle", "DQP" })); + } + +} From 94b3782db7a434772bd0dbea606b12beeef1fca5 Mon Sep 17 00:00:00 2001 From: Jinsheng Ba <43855029@qq.com> Date: Wed, 20 Mar 2024 10:56:38 +0800 Subject: [PATCH 409/856] add doc (#926) --- README.md | 5 +++-- docs/PAPERS.md | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d84a993b1..83e207485 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,8 @@ SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Man SQLancer operates in the following two phases: -1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. **News: we support Query Plan Guidance (QPG) now. See Generation Approaches below.** -2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. **News: we support Cardinality Estimation Restriction Testing (CERT) oracle now. See Testing Approaches below.** +1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. +2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. **News: we support Differential Query Plans (DQP) oracle now. See Testing Approaches below.** # Getting Started @@ -41,6 +41,7 @@ If you launch SQLancer without parameters, available options and commands are di | Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | | Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | | Cardinality Estimation Restriction Testing (CERT) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. It is described [here](https://arxiv.org/abs/2306.00355). It derives a query to a more restrict query, whose estimated cardinality should be no more than that for the original query. An violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. | +| Differential Query Plans (DQP) | DQP aims to find logic bugs in database systems by checking whether the query plans of the same query perform consistently. It is described [here](https://bajinsheng.github.io/assets/pdf/dqp_sigmod24.pdf). DQP supports MySQL, MariaDB, and TiDB.| # Generation Approaches | Approach | Description | diff --git a/docs/PAPERS.md b/docs/PAPERS.md index c0e13dd3e..a42b42c12 100644 --- a/docs/PAPERS.md +++ b/docs/PAPERS.md @@ -65,6 +65,37 @@ This paper describes Query Plan Guidance (QPG), a test case generation method gu } ``` +## CERT: Finding Performance Issues in Database Systems Through the Lens of Cardinality Estimation + +This paper describes CERT, a testing approach to find performance issues by inspecting inconsistent estimated cardinalities. A preprint is available [here](https://bajinsheng.github.io/assets/pdf/cert_icse24.pdf). + +``` +@inproceedings{cert, + author = {Ba, Jinsheng and Rigger, Manuel}, + title = {CERT: Finding Performance Issues in Database Systems Through the Lens of Cardinality Estimation}, + booktitle = {The 46th International Conference on Software Engineering (ICSE'24)}, + year = {2024}, + month = apr, +} +``` + +## Keep It Simple: Testing Databases via Differential Query Plans + +This paper describes DQP, a testing approach to find logic bugs in database systems by comparing the query plans of different database systems. A preprint is available [here](https://bajinsheng.github.io/assets/pdf/dqp_sigmod24.pdf). + +``` +@article{dqp, + author = {Ba, Jinsheng and Rigger, Manuel}, + title = {Keep It Simple: Testing Databases via Differential Query Plans}, + year = {2024}, + issue_date = {June 2024}, + publisher = {Association for Computing Machinery}, + address = {New York, NY, USA}, + journal = {Proceeding of ACM Management of Data (SIGMOD'24)}, + month = jun +} +``` + # Comparing SQLancer With Other Tools that Find Logic Bugs If you want to fairly compare other tools with SQLancer, we would be glad to provide feedback (e.g., feel free to send an email to manuel.rigger@inf.ethz.ch). We have the following general recommendations and comments: From 91a3a9b60ba81c57637b0f8f7bc673609736dfa3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 29 Feb 2024 17:59:39 +0800 Subject: [PATCH 410/856] Add interfaces for Expression --- src/sqlancer/common/ast/newast/Constant.java | 6 +++ .../common/ast/newast/Expression.java | 6 +++ src/sqlancer/common/ast/newast/Join.java | 14 ++++++ src/sqlancer/common/ast/newast/Select.java | 50 +++++++++++++++++++ 4 files changed, 76 insertions(+) create mode 100644 src/sqlancer/common/ast/newast/Constant.java create mode 100644 src/sqlancer/common/ast/newast/Expression.java create mode 100644 src/sqlancer/common/ast/newast/Join.java create mode 100644 src/sqlancer/common/ast/newast/Select.java diff --git a/src/sqlancer/common/ast/newast/Constant.java b/src/sqlancer/common/ast/newast/Constant.java new file mode 100644 index 000000000..a6a6dfd49 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Constant.java @@ -0,0 +1,6 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTableColumn; + +public interface Constant> extends Expression { +} diff --git a/src/sqlancer/common/ast/newast/Expression.java b/src/sqlancer/common/ast/newast/Expression.java new file mode 100644 index 000000000..925cb82c5 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Expression.java @@ -0,0 +1,6 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTableColumn; + +public interface Expression> { +} diff --git a/src/sqlancer/common/ast/newast/Join.java b/src/sqlancer/common/ast/newast/Join.java new file mode 100644 index 000000000..b1d1ed280 --- /dev/null +++ b/src/sqlancer/common/ast/newast/Join.java @@ -0,0 +1,14 @@ +package sqlancer.common.ast.newast; + +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; + +public interface Join, T extends AbstractTable, C extends AbstractTableColumn> + extends Expression { + + T getTable(); + + Expression getOnClause(); + + void setOnClause(E onClause); +} diff --git a/src/sqlancer/common/ast/newast/Select.java b/src/sqlancer/common/ast/newast/Select.java new file mode 100644 index 000000000..e96501b2b --- /dev/null +++ b/src/sqlancer/common/ast/newast/Select.java @@ -0,0 +1,50 @@ +package sqlancer.common.ast.newast; + +import java.util.List; + +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; + +public interface Select, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> + extends Expression { + + void setFromTables(List fromTables); + + List getFromList(); + + void setFromList(List fromList); + + Expression getWhereClause(); + + void setWhereClause(E whereClause); + + void setGroupByClause(List groupByClause); + + List getGroupByClause(); + + void setLimitClause(E limitClause); + + Expression getLimitClause(); + + List getOrderByClause(); + + void setOrderByExpressions(List orderBy); + + void setOffsetClause(E offsetClause); + + Expression getOffsetClause(); + + void setFetchColumns(List fetchColumns); + + List getFetchColumns(); + + void setJoinClauses(List joinStatements); + + List getJoinClauses(); + + void setHavingClause(E havingClause); + + Expression getHavingClause(); + + String asString(); +} From ab066addf7042cb4a6da0cc6439ba345809ecfb9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 25 Mar 2024 10:34:48 +0800 Subject: [PATCH 411/856] Change setFromTables to setFromList for SQLite and MariaDB --- src/sqlancer/common/ast/newast/Select.java | 2 -- .../mariadb/ast/MariaDBSelectStatement.java | 10 ------- .../mariadb/ast/MariaDBStringVisitor.java | 13 +++++++-- .../mariadb/ast/MariaDBTableReference.java | 16 +++++++++++ src/sqlancer/mariadb/ast/MariaDBVisitor.java | 4 +++ .../mariadb/oracle/MariaDBDQPOracle.java | 4 ++- .../mariadb/oracle/MariaDBNoRECOracle.java | 5 ++-- src/sqlancer/sqlite3/ast/SQLite3Select.java | 4 --- .../sqlite3/oracle/SQLite3NoRECOracle.java | 2 +- .../SQLite3PivotedQuerySynthesisOracle.java | 2 +- .../sqlite3/oracle/tlp/SQLite3TLPBase.java | 2 +- .../oracle/tlp/SQLite3TLPHavingOracle.java | 2 +- test/sqlancer/dbms/TestMariaDB.java | 28 +++++++++++++++++++ 13 files changed, 69 insertions(+), 25 deletions(-) create mode 100644 src/sqlancer/mariadb/ast/MariaDBTableReference.java diff --git a/src/sqlancer/common/ast/newast/Select.java b/src/sqlancer/common/ast/newast/Select.java index e96501b2b..0953f289a 100644 --- a/src/sqlancer/common/ast/newast/Select.java +++ b/src/sqlancer/common/ast/newast/Select.java @@ -8,8 +8,6 @@ public interface Select, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> extends Expression { - void setFromTables(List fromTables); - List getFromList(); void setFromList(List fromList); diff --git a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java index 83b944a17..a690276d1 100644 --- a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java +++ b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java @@ -4,7 +4,6 @@ import java.util.List; import sqlancer.common.ast.SelectBase; -import sqlancer.mariadb.MariaDBSchema.MariaDBTable; public class MariaDBSelectStatement extends SelectBase implements MariaDBExpression { @@ -14,7 +13,6 @@ public enum MariaDBSelectType { private List groupBys = new ArrayList<>(); private List columns = new ArrayList<>(); - private List tables = new ArrayList<>(); private MariaDBSelectType selectType = MariaDBSelectType.ALL; private MariaDBExpression whereCondition; @@ -28,10 +26,6 @@ public void setFetchColumns(List columns) { } - public void setFromTables(List tables) { - this.tables = tables; - } - public void setSelectType(MariaDBSelectType selectType) { this.selectType = selectType; } @@ -53,10 +47,6 @@ public MariaDBSelectType getSelectType() { return selectType; } - public List getTables() { - return tables; - } - public MariaDBExpression getWhereCondition() { return whereCondition; } diff --git a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java index 07951d9ac..17c84ef4a 100644 --- a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.mariadb.ast; import java.util.List; -import java.util.stream.Collectors; public class MariaDBStringVisitor extends MariaDBVisitor { @@ -41,8 +40,13 @@ public void visit(MariaDBSelectStatement s) { visit(column); } sb.append(" FROM "); - sb.append(s.getTables().stream().map(t -> t.getName()).collect(Collectors.joining(", "))); + for (int j = 0; j < s.getFromList().size(); j++) { + if (j != 0) { + sb.append(", "); + } + visit(s.getFromList().get(j)); + } for (MariaDBExpression j : s.getJoinList()) { visit(j); } @@ -167,4 +171,9 @@ public void visit(MariaDBJoin join) { visit(join.getOnClause()); } } + + @Override + public void visit(MariaDBTableReference ref) { + sb.append(ref.getTable().getName()); + } } diff --git a/src/sqlancer/mariadb/ast/MariaDBTableReference.java b/src/sqlancer/mariadb/ast/MariaDBTableReference.java new file mode 100644 index 000000000..f045907ca --- /dev/null +++ b/src/sqlancer/mariadb/ast/MariaDBTableReference.java @@ -0,0 +1,16 @@ +package sqlancer.mariadb.ast; + +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; + +public class MariaDBTableReference implements MariaDBExpression { + + private final MariaDBTable table; + + public MariaDBTableReference(MariaDBTable table) { + this.table = table; + } + + public MariaDBTable getTable() { + return table; + } +} diff --git a/src/sqlancer/mariadb/ast/MariaDBVisitor.java b/src/sqlancer/mariadb/ast/MariaDBVisitor.java index af94394b9..8626dc967 100644 --- a/src/sqlancer/mariadb/ast/MariaDBVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBVisitor.java @@ -24,6 +24,8 @@ public abstract class MariaDBVisitor { public abstract void visit(MariaDBJoin join); + public abstract void visit(MariaDBTableReference join); + public void visit(MariaDBExpression expr) { if (expr instanceof MariaDBConstant) { visit((MariaDBConstant) expr); @@ -47,6 +49,8 @@ public void visit(MariaDBExpression expr) { visit((MariaDBInOperation) expr); } else if (expr instanceof MariaDBJoin) { visit((MariaDBJoin) expr); + } else if (expr instanceof MariaDBTableReference) { + visit((MariaDBTableReference) expr); } else { throw new AssertionError(expr.getClass()); } diff --git a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java index a92be5d1e..ee85f2df7 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java @@ -17,6 +17,7 @@ import sqlancer.mariadb.ast.MariaDBExpression; import sqlancer.mariadb.ast.MariaDBJoin; import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBTableReference; import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; import sqlancer.mariadb.gen.MariaDBSetGenerator; @@ -60,7 +61,8 @@ public void check() throws Exception { select.setJoinList(joinExpressions.stream().map(j -> (MariaDBExpression) j).collect(Collectors.toList())); // Set the from clause from the tables that are not used in the join. - select.setFromTables(tables.getTables()); + select.setFromList( + tables.getTables().stream().map(t -> new MariaDBTableReference(t)).collect(Collectors.toList())); // Get the result of the first query String originalQueryString = MariaDBVisitor.asString(select); diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index bba0627e4..ab6438275 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -23,6 +23,7 @@ import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation.MariaDBPostfixUnaryOperator; import sqlancer.mariadb.ast.MariaDBSelectStatement; import sqlancer.mariadb.ast.MariaDBSelectStatement.MariaDBSelectType; +import sqlancer.mariadb.ast.MariaDBTableReference; import sqlancer.mariadb.ast.MariaDBText; import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; @@ -76,7 +77,7 @@ private int getUnoptimizedQuery(MariaDBTable randomTable, MariaDBExpression rand randomWhereCondition); MariaDBText asText = new MariaDBText(isTrue, " as count", false); select.setFetchColumns(Arrays.asList(asText)); - select.setFromTables(Arrays.asList(randomTable)); + select.setFromList(Arrays.asList(new MariaDBTableReference(randomTable))); select.setSelectType(MariaDBSelectType.ALL); int secondCount = 0; @@ -103,7 +104,7 @@ private int getOptimizedQuery(MariaDBTable randomTable, MariaDBExpression random new MariaDBColumnName(new MariaDBColumn("*", MariaDBDataType.INT, false, 0)), MariaDBAggregateFunction.COUNT); select.setFetchColumns(Arrays.asList(aggr)); - select.setFromTables(Arrays.asList(randomTable)); + select.setFromList(Arrays.asList(new MariaDBTableReference(randomTable))); select.setWhereClause(randomWhereCondition); select.setSelectType(MariaDBSelectType.ALL); int firstCount; diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index bff7fb775..5ff008709 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -46,10 +46,6 @@ public void setSelectType(SelectType fromOptions) { this.setFromOptions(fromOptions); } - public void setFromTables(List fromTables) { - this.setFromList(fromTables); - } - public SelectType getFromOptions() { return fromOptions; } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 174efcbb6..45ec6631d 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -80,7 +80,7 @@ public void check() throws SQLException { List joinStatements = gen.getRandomJoinClauses(tables); List tableRefs = SQLite3Common.getTableRefs(tables, s); SQLite3Select select = new SQLite3Select(); - select.setFromTables(tableRefs); + select.setFromList(tableRefs); select.setJoinClauses(joinStatements); Function optimizedQuery = getOptimizedQuery(select, randomWhereCondition); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index c19a1059d..4021c88e9 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -75,7 +75,7 @@ public SQLite3Select getQuery() throws SQLException { .filter(c -> !SQLite3Schema.ROWID_STRINGS.contains(c.getName())).collect(Collectors.toList()); List joinStatements = getJoinStatements(globalState, tables, columnsWithoutRowid); selectStatement.setJoinClauses(joinStatements); - selectStatement.setFromTables(SQLite3Common.getTableRefs(tables, globalState.getSchema())); + selectStatement.setFromList(SQLite3Common.getTableRefs(tables, globalState.getSchema())); fetchColumns = Randomly.nonEmptySubset(columnsWithoutRowid); List allTables = new ArrayList<>(); diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java index 650df8537..b946ebe38 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPBase.java @@ -48,7 +48,7 @@ public void check() throws SQLException { List joinStatements = gen.getRandomJoinClauses(tables); List tableRefs = SQLite3Common.getTableRefs(tables, s); select.setJoinClauses(joinStatements.stream().collect(Collectors.toList())); - select.setFromTables(tableRefs); + select.setFromList(tableRefs); select.setWhereClause(null); } diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java index 718c0d4ce..248d4db5f 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPHavingOracle.java @@ -57,7 +57,7 @@ public void check() throws SQLException { List from = SQLite3Common.getTableRefs(tables, state.getSchema()); select.setJoinClauses(joinStatements); select.setSelectType(SelectType.ALL); - select.setFromTables(from); + select.setFromList(from); // TODO order by? select.setGroupByClause(groupByColumns); select.setHavingClause(null); diff --git a/test/sqlancer/dbms/TestMariaDB.java b/test/sqlancer/dbms/TestMariaDB.java index e5c188d28..f1b86a339 100644 --- a/test/sqlancer/dbms/TestMariaDB.java +++ b/test/sqlancer/dbms/TestMariaDB.java @@ -3,12 +3,40 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assumptions.assumeTrue; +import java.util.List; +import java.util.stream.Collectors; + import org.junit.jupiter.api.Test; import sqlancer.Main; +import sqlancer.mariadb.MariaDBSchema; +import sqlancer.mariadb.ast.MariaDBColumnName; +import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBTableReference; +import sqlancer.mariadb.ast.MariaDBVisitor; public class TestMariaDB { + @Test + public void testSelectAsString() { + MariaDBSchema.MariaDBColumn c0 = new MariaDBSchema.MariaDBColumn("c0", MariaDBSchema.MariaDBDataType.INT, true, + 0); + MariaDBSchema.MariaDBColumn c1 = new MariaDBSchema.MariaDBColumn("c1", MariaDBSchema.MariaDBDataType.INT, false, + 0); + List columns = List.of(c0, c1); + List indices = List.of(); + MariaDBSchema.MariaDBTable t1 = new MariaDBSchema.MariaDBTable("t1", columns, indices, + MariaDBSchema.MariaDBTable.MariaDBEngine.INNO_DB); + MariaDBSchema.MariaDBTables tables = new MariaDBSchema.MariaDBTables(List.of(t1)); + + MariaDBSelectStatement select = new MariaDBSelectStatement(); + select.setFetchColumns(tables.getColumns().stream().map(MariaDBColumnName::new).collect(Collectors.toList())); + select.setFromList(tables.getTables().stream().map(MariaDBTableReference::new).collect(Collectors.toList())); + + String selectString = MariaDBVisitor.asString(select); + assertEquals("SELECT c0, c1 FROM t1", selectString); + } + @Test public void testMariaDB() { String mariaDBAvailable = System.getenv("MARIADB_AVAILABLE"); From 3da3faf6238ca008ff3f5edd3548f2cabcf9cfce Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Mon, 25 Mar 2024 11:33:24 +0800 Subject: [PATCH 412/856] Update expected errors (#922) * Refactor expected errors * Update creation of ExpectedErrors * Expand ExpectedErrors unit tests * Add missing errors to postgres * Update expected errors for sqlite3 --- src/sqlancer/citus/gen/CitusCommon.java | 11 +- src/sqlancer/clickhouse/ClickHouseErrors.java | 124 +++++++--------- src/sqlancer/cnosdb/CnosDBExpectedError.java | 116 +++++++-------- .../cockroachdb/CockroachDBErrors.java | 68 +++++++-- src/sqlancer/common/query/ExpectedErrors.java | 64 +++++++-- src/sqlancer/databend/DatabendErrors.java | 33 ++++- src/sqlancer/doris/DorisErrors.java | 23 ++- src/sqlancer/duckdb/DuckDBErrors.java | 53 +++++-- src/sqlancer/h2/H2Errors.java | 27 +++- src/sqlancer/hsqldb/HSQLDBErrors.java | 35 ++++- src/sqlancer/mariadb/MariaDBErrors.java | 32 ++++- .../materialize/gen/MaterializeCommon.java | 109 +++++++++++--- src/sqlancer/mysql/MySQLErrors.java | 23 ++- src/sqlancer/oceanbase/OceanBaseErrors.java | 22 ++- src/sqlancer/postgres/gen/PostgresCommon.java | 133 ++++++++++++++---- src/sqlancer/presto/PrestoErrors.java | 48 ++++++- src/sqlancer/questdb/QuestDBErrors.java | 31 +++- src/sqlancer/sqlite3/SQLite3Errors.java | 71 +++++++++- src/sqlancer/stonedb/StoneDBErrors.java | 43 ++++-- src/sqlancer/tidb/TiDBErrors.java | 33 ++++- .../timescaledb/gen/TimescaleDBCommon.java | 9 +- src/sqlancer/yugabyte/ycql/YCQLErrors.java | 13 +- src/sqlancer/yugabyte/ysql/YSQLErrors.java | 113 +++++++++++++-- test/sqlancer/TestExpectedErrors.java | 72 ++++++++++ 24 files changed, 1028 insertions(+), 278 deletions(-) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index f4a121efe..1ca2af138 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -1,5 +1,8 @@ package sqlancer.citus.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.citus.CitusBugs; import sqlancer.common.query.ExpectedErrors; @@ -8,8 +11,9 @@ public final class CitusCommon { private CitusCommon() { } - public static void addCitusErrors(ExpectedErrors errors) { + public static List getCitusErrors() { // not supported by Citus + ArrayList errors = new ArrayList<>(); errors.add("failed to evaluate partition key in insert"); errors.add("cannot perform an INSERT without a partition column value"); errors.add("cannot perform an INSERT with NULL in the partition column"); @@ -74,6 +78,11 @@ public static void addCitusErrors(ExpectedErrors errors) { if (CitusBugs.bug4079) { errors.add("aggregate function calls cannot be nested"); } + + return errors; } + public static void addCitusErrors(ExpectedErrors errors) { + errors.addAll(getCitusErrors()); + } } diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index e824faef9..23c27c8bf 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -1,5 +1,7 @@ package sqlancer.clickhouse; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class ClickHouseErrors { @@ -7,79 +9,57 @@ public final class ClickHouseErrors { private ClickHouseErrors() { } - public static void addExpectedExpressionErrors(ExpectedErrors errors) { - errors.add("Argument at index 1 for function like must be constant"); - errors.add("Argument at index 1 for function notLike must be constant"); - errors.add("Attempt to read after eof: while converting"); - errors.add("Bad get: has Int64, requested UInt64"); - errors.add("Cannot convert string"); - errors.add("Cannot insert NULL value into a column of type"); - errors.add("Cannot parse Int32 from String, because value is too short"); - errors.add("Cannot parse NaN.: while converting"); // https://github.com/ClickHouse/ClickHouse/issues/22710 - errors.add("Cannot parse infinity."); - errors.add("Cannot parse number with a sign character but without any numeric character"); - errors.add("Cannot parse number with multiple sign (+/-) characters or intermediate sign character"); - errors.add("Cannot parse string"); - errors.add("Cannot read floating point value"); - errors.add("Cyclic aliases: default expression and column type are incompatible"); - errors.add("Directory for table data"); - errors.add("Directory not empty"); - errors.add("Expected one of: compound identifier, identifier, list of elements (version"); // VALUES () - errors.add("Function 'like' doesn't support search with non-constant needles in constant haystack"); - errors.add("Illegal type"); - errors.add("Illegal value (aggregate function) for positional argument in GROUP BY"); - errors.add("Invalid escape sequence at the end of LIKE pattern"); - errors.add("Invalid type for filter in"); - errors.add("Memory limit"); - errors.add("OptimizedRegularExpression: cannot compile re2"); - errors.add("Partition key cannot contain constants"); - errors.add("Positional argument out of bounds"); - errors.add("Sampling expression must be present in the primary key"); - errors.add("Sorting key cannot contain constants"); - errors.add("There is no supertype for types"); - errors.add("argument of function"); - errors.add("but its arguments considered equal according to constraints"); - errors.add("does not return a value of type UInt8"); - errors.add("doesn't exist"); // TODO: consecutive test runs can lead to dropped database - errors.add("in block. There are only columns:"); // https://github.com/ClickHouse/ClickHouse/issues/42399 - errors.add("invalid character class range"); - errors.add("invalid escape sequence"); - errors.add("is not under aggregate function and not in GROUP BY"); - errors.add("is not under aggregate function"); - errors.add("is violated at row 1. Expression:"); // TODO: check constraint on table creation - errors.add( - "is violated, because it is a constant expression returning 0. It is most likely an error in table definition"); - errors.add("there are only columns"); - errors.add("there are columns"); - errors.add("in block. (NOT_FOUND_COLUMN_IN_BLOCK)"); - errors.add("Missing columns"); - errors.add("Ambiguous column"); - errors.add("Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)"); - errors.add("Floating point partition key is not supported"); - errors.add("Cannot get JOIN keys from JOIN ON section"); - errors.add("ILLEGAL_DIVISION"); - errors.add("DECIMAL_OVERFLOW"); - errors.add("Cannot convert out of range floating point value to integer type"); - errors.add("Unexpected inf or nan to integer conversion"); - errors.add("No such name in Block::erase"); // https://github.com/ClickHouse/ClickHouse/issues/42769 - errors.add("EMPTY_LIST_OF_COLUMNS_QUERIED"); // https://github.com/ClickHouse/ClickHouse/issues/43003 - errors.add("cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)"); - errors.add("AMBIGUOUS_IDENTIFIER"); - errors.add("CYCLIC_ALIASES"); - errors.add("Positional argument numeric constant expression is not representable as"); - errors.add("Positional argument must be constant with numeric type"); - errors.add(" is out of bounds. Expected in range"); - errors.add("with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)"); - errors.add("Unexpected inf or nan to integer conversion"); - errors.add("Unsigned type must not contain"); - errors.add("Unexpected inf or nan to integer conversion"); + public static List getExpectedExpressionErrors() { + return List.of("Argument at index 1 for function like must be constant", + "Argument at index 1 for function notLike must be constant", + "Attempt to read after eof: while converting", "Bad get: has Int64, requested UInt64", + "Cannot convert string", "Cannot insert NULL value into a column of type", + "Cannot parse Int32 from String, because value is too short", "Cannot parse NaN.: while converting", // https://github.com/ClickHouse/ClickHouse/issues/22710 + "Cannot parse infinity.", "Cannot parse number with a sign character but without any numeric character", + "Cannot parse number with multiple sign (+/-) characters or intermediate sign character", + "Cannot parse string", "Cannot read floating point value", + "Cyclic aliases: default expression and column type are incompatible", "Directory for table data", + "Directory not empty", "Expected one of: compound identifier, identifier, list of elements (version", // VALUES + // () + "Function 'like' doesn't support search with non-constant needles in constant haystack", "Illegal type", + "Illegal value (aggregate function) for positional argument in GROUP BY", + "Invalid escape sequence at the end of LIKE pattern", "Invalid type for filter in", "Memory limit", + "OptimizedRegularExpression: cannot compile re2", "Partition key cannot contain constants", + "Positional argument out of bounds", "Sampling expression must be present in the primary key", + "Sorting key cannot contain constants", "There is no supertype for types", "argument of function", + "but its arguments considered equal according to constraints", "does not return a value of type UInt8", + "doesn't exist", // TODO: consecutive test runs can lead to dropped database + "in block. There are only columns:", // https://github.com/ClickHouse/ClickHouse/issues/42399 + "invalid character class range", "invalid escape sequence", + "is not under aggregate function and not in GROUP BY", "is not under aggregate function", + "is violated at row 1. Expression:", // TODO: check constraint on table creation + "is violated, because it is a constant expression returning 0. It is most likely an error in table definition", + "there are only columns", "there are columns", "in block. (NOT_FOUND_COLUMN_IN_BLOCK)", + "Missing columns", "Ambiguous column", + "Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)", + "Floating point partition key is not supported", "Cannot get JOIN keys from JOIN ON section", + "ILLEGAL_DIVISION", "DECIMAL_OVERFLOW", + "Cannot convert out of range floating point value to integer type", + "Unexpected inf or nan to integer conversion", "No such name in Block::erase", // https://github.com/ClickHouse/ClickHouse/issues/42769 + "EMPTY_LIST_OF_COLUMNS_QUERIED", // https://github.com/ClickHouse/ClickHouse/issues/43003 + "cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)", "AMBIGUOUS_IDENTIFIER", "CYCLIC_ALIASES", + "Positional argument numeric constant expression is not representable as", + "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", + "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)", + "Unexpected inf or nan to integer conversion", "Unsigned type must not contain", + "Unexpected inf or nan to integer conversion", - // The way we generate JOINs we can have ambiguous left table column without alias - // We may not count it as an issue, but it makes no sense to add more complex AST generation logic - errors.add("MULTIPLE_EXPRESSIONS_FOR_ALIAS"); - errors.add("AMBIGUOUS_IDENTIFIER"); // https://github.com/ClickHouse/ClickHouse/issues/45389 - errors.add("AMBIGUOUS_COLUMN_NAME"); // same https://github.com/ClickHouse/ClickHouse/issues/45389 - errors.add("Cannot parse number with multiple sign"); + // The way we generate JOINs we can have ambiguous left table column without + // alias + // We may not count it as an issue, but it makes no sense to add more complex + // AST generation logic + "MULTIPLE_EXPRESSIONS_FOR_ALIAS", "AMBIGUOUS_IDENTIFIER", // https://github.com/ClickHouse/ClickHouse/issues/45389 + "AMBIGUOUS_COLUMN_NAME", // same https://github.com/ClickHouse/ClickHouse/issues/45389 + "Cannot parse number with multiple sign"); + } + + public static void addExpectedExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpectedExpressionErrors()); } } diff --git a/src/sqlancer/cnosdb/CnosDBExpectedError.java b/src/sqlancer/cnosdb/CnosDBExpectedError.java index 516e44ce4..5932bfdac 100644 --- a/src/sqlancer/cnosdb/CnosDBExpectedError.java +++ b/src/sqlancer/cnosdb/CnosDBExpectedError.java @@ -6,74 +6,76 @@ import sqlancer.common.query.ExpectedErrors; public final class CnosDBExpectedError { - private static final List ERRORS = new ArrayList<>(); private CnosDBExpectedError() { } - static { - ERRORS.add("have the same name. Consider aliasing"); - ERRORS.add( + public static List getExpectedErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("have the same name. Consider aliasing"); + errors.add( "error: Optimizer rule 'projection_push_down' failed due to unexpected error: Schema error: Schema contains duplicate qualified field name"); - ERRORS.add("Projection references non-aggregate values:"); - ERRORS.add("External err: Schema error: No field named"); - ERRORS.add( + errors.add("Projection references non-aggregate values:"); + errors.add("External err: Schema error: No field named"); + errors.add( "Optimizer rule 'common_sub_expression_eliminate' failed due to unexpected error: Schema error: No field named"); - ERRORS.add("Binary"); - ERRORS.add("Invalid pattern in LIKE expression"); - ERRORS.add("If the projection contains the time column, it must contain the field column."); - ERRORS.add("Schema error: No field named"); - ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:"); - ERRORS.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error"); - ERRORS.add("Schema error: No field named "); - ERRORS.add("err: External err: Schema error: No field named"); - ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error"); - ERRORS.add("Csv error: CSV Writer does not support List"); - ERRORS.add("This feature is not implemented: cross join."); - ERRORS.add("Execution error: field position must be greater than zero"); - ERRORS.add("First argument of `DATE_PART` must be non-null scalar Utf8"); - ERRORS.add("Cannot create filter with non-boolean predicate 'NULL' returning Null"); - ERRORS.add("requested character too large for encoding."); - ERRORS.add("Can not find compatible types to compare Boolean with [Utf8]."); - ERRORS.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT"); - ERRORS.add("HAVING clause references non-aggregate values:"); - ERRORS.add("Cannot create filter with non-boolean predicate"); - ERRORS.add("negative substring length not allowed"); - ERRORS.add("The function Sum does not support inputs of type Boolean."); - ERRORS.add("The function Avg does not support inputs of type Boolean."); - ERRORS.add("Percentile value must be between 0.0 and 1.0 inclusive"); - ERRORS.add("Date part '' not supported"); - ERRORS.add("Min/Max accumulator not implemented for type Boolean."); - ERRORS.add("meta need get_series_id_by_filter"); - ERRORS.add("Arrow: Cast error:"); - ERRORS.add("Arrow error: Cast error:"); - ERRORS.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:"); - ERRORS.add("Arrow error: Divide by zero error"); - ERRORS.add("desired percentile argument must be float literal"); - ERRORS.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)"); - ERRORS.add("Execution error: Date part"); - ERRORS.add("Physical plan does not support logical expression MIN(Boolean"); - ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64"); - ERRORS.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64."); - ERRORS.add("Data type UInt64 not supported for binary operation '#' on dyn arrays."); - ERRORS.add("Arrow: Divide by zero error"); - ERRORS.add("The function ApproxPercentileCont does not support inputs of type Null."); - ERRORS.add("can't be evaluated because there isn't a common type to coerce the types to"); - ERRORS.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug"); - ERRORS.add("The function ApproxMedian does not support inputs of type Null."); - ERRORS.add("null character not permitted."); - ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Null."); - ERRORS.add("This feature is not implemented"); - ERRORS.add("The function Avg does not support inputs of type Null."); - ERRORS.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]"); - ERRORS.add( + errors.add("Binary"); + errors.add("Invalid pattern in LIKE expression"); + errors.add("If the projection contains the time column, it must contain the field column."); + errors.add("Schema error: No field named"); + errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:"); + errors.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error"); + errors.add("Schema error: No field named "); + errors.add("err: External err: Schema error: No field named"); + errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error"); + errors.add("Csv error: CSV Writer does not support List"); + errors.add("This feature is not implemented: cross join."); + errors.add("Execution error: field position must be greater than zero"); + errors.add("First argument of `DATE_PART` must be non-null scalar Utf8"); + errors.add("Cannot create filter with non-boolean predicate 'NULL' returning Null"); + errors.add("requested character too large for encoding."); + errors.add("Can not find compatible types to compare Boolean with [Utf8]."); + errors.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT"); + errors.add("HAVING clause references non-aggregate values:"); + errors.add("Cannot create filter with non-boolean predicate"); + errors.add("negative substring length not allowed"); + errors.add("The function Sum does not support inputs of type Boolean."); + errors.add("The function Avg does not support inputs of type Boolean."); + errors.add("Percentile value must be between 0.0 and 1.0 inclusive"); + errors.add("Date part '' not supported"); + errors.add("Min/Max accumulator not implemented for type Boolean."); + errors.add("meta need get_series_id_by_filter"); + errors.add("Arrow: Cast error:"); + errors.add("Arrow error: Cast error:"); + errors.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:"); + errors.add("Arrow error: Divide by zero error"); + errors.add("desired percentile argument must be float literal"); + errors.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)"); + errors.add("Execution error: Date part"); + errors.add("Physical plan does not support logical expression MIN(Boolean"); + errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64"); + errors.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64."); + errors.add("Data type UInt64 not supported for binary operation '#' on dyn arrays."); + errors.add("Arrow: Divide by zero error"); + errors.add("The function ApproxPercentileCont does not support inputs of type Null."); + errors.add("can't be evaluated because there isn't a common type to coerce the types to"); + errors.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug"); + errors.add("The function ApproxMedian does not support inputs of type Null."); + errors.add("null character not permitted."); + errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Null."); + errors.add("This feature is not implemented"); + errors.add("The function Avg does not support inputs of type Null."); + errors.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]"); + errors.add( "Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed."); - ERRORS.add("Coercion from"); + errors.add("Coercion from"); + return errors; } public static ExpectedErrors expectedErrors() { ExpectedErrors res = new ExpectedErrors(); - res.addAll(ERRORS); + res.addAll(getExpectedErrors()); return res; } diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 7cb738ee8..63d0b30ce 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.cockroachdb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class CockroachDBErrors { @@ -7,7 +10,9 @@ public final class CockroachDBErrors { private CockroachDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add(" non-streaming operator encountered when vectorize=auto"); if (CockroachDBBugs.bug46915) { @@ -105,10 +110,10 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("as type time"); errors.add("as TimeTZ"); errors.add("as type decimal"); - addIntervalTypeErrors(errors); - addFunctionErrors(errors); - addGroupByErrors(errors); - addJoinTypes(errors); + errors.addAll(getIntervalTypeErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getGroupByErrors()); + errors.addAll(getJoinTypes()); errors.add("as int4, found type: decimal"); errors.add("to be of type int2, found type decimal"); errors.add("to be of type int, found type decimal"); // arithmetic overflows @@ -210,11 +215,19 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("argument of OFFSET must be type int, not type decimal"); errors.add("ERROR: for SELECT DISTINCT, ORDER BY expressions must appear in select list"); - addArrayErrors(errors); - addComputedColumnErrors(errors); + errors.addAll(getArrayErrors()); + errors.addAll(getComputedColumnErrors()); + + return errors; } - private static void addArrayErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + private static List getArrayErrors() { + ArrayList errors = new ArrayList<>(); + // arrays errors.add("cannot determine type of empty array"); errors.add("unknown signature: max(unknown[])"); @@ -260,18 +273,29 @@ private static void addArrayErrors(ExpectedErrors errors) { errors.add("to be of type int[], found type decimal[]"); errors.add("to be of type unknown[]"); // IF with null array + + return errors; } - private static void addIntervalTypeErrors(ExpectedErrors errors) { + private static List getIntervalTypeErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("overflow during Encode"); errors.add("type interval"); + + return errors; } - private static void addJoinTypes(ExpectedErrors errors) { + private static List getJoinTypes() { + ArrayList errors = new ArrayList<>(); + errors.add("JOIN/USING types"); + + return errors; } - private static void addGroupByErrors(ExpectedErrors errors) { + private static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); errors.add("non-integer constant in GROUP BY"); // https://github.com/cockroachdb/cockroach/pull/46649 -> aggregates on NULL are @@ -293,9 +317,11 @@ private static void addGroupByErrors(ExpectedErrors errors) { errors.add("unknown signature: abs(string)"); errors.add("unknown signature: acos(string)"); + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); // functions errors.add("abs of min integer value (-9223372036854775808) not defined"); // ABS errors.add("the input string must not be empty"); // ASCII @@ -311,15 +337,29 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("substring(): negative substring length"); // substring errors.add("negative substring length"); // substring errors.add("must be greater than zero"); // split_part + + return errors; } - public static void addTransactionErrors(ExpectedErrors errors) { + public static List getTransactionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("current transaction is aborted"); + + return errors; } - private static void addComputedColumnErrors(ExpectedErrors errors) { + public static void addTransactionErrors(ExpectedErrors errors) { + errors.addAll(getTransactionErrors()); + } + + private static List getComputedColumnErrors() { + ArrayList errors = new ArrayList<>(); + // computed columns errors.add("computed column expressions cannot reference computed columns"); + + return errors; } } diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index 1016969c9..348da179f 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -1,9 +1,8 @@ package sqlancer.common.query; -import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashSet; -import java.util.List; import java.util.Set; import java.util.regex.Pattern; @@ -14,8 +13,18 @@ */ public class ExpectedErrors { - private final Set errors = new HashSet<>(); - private final List regexes = new ArrayList<>(); + private final Set errors; + private final Set regexes; + + public ExpectedErrors() { + this.errors = new HashSet<>(); + this.regexes = new HashSet<>(); + } + + public ExpectedErrors(Collection errors, Collection regexErrors) { + this.errors = new HashSet<>(errors); + this.regexes = new HashSet<>(regexErrors); + } public ExpectedErrors add(String error) { if (error == null) { @@ -33,6 +42,11 @@ public ExpectedErrors addRegex(Pattern errorPattern) { return this; } + public ExpectedErrors addRegexString(String errorPattern) { + regexes.add(Pattern.compile(errorPattern)); + return this; + } + public ExpectedErrors addAll(Collection list) { errors.addAll(list); return this; @@ -46,12 +60,19 @@ public ExpectedErrors addAllRegexes(Collection list) { return this; } - public static ExpectedErrors from(String... errors) { - ExpectedErrors expectedErrors = new ExpectedErrors(); - for (String error : errors) { - expectedErrors.add(error); + public ExpectedErrors addAllRegexStrings(Collection list) { + for (String error : list) { + regexes.add(Pattern.compile(error)); } - return expectedErrors; + return this; + } + + public static ExpectedErrors from(String... errors) { + return newErrors().with(errors).build(); + } + + public static ExpectedErrorsBuilder newErrors() { + return new ExpectedErrorsBuilder(); } /** @@ -80,4 +101,29 @@ public boolean errorIsExpected(String error) { return false; } + public static class ExpectedErrorsBuilder { + private final Set errors = new HashSet<>(); + private final Set regexes = new HashSet<>(); + + public ExpectedErrorsBuilder with(String... list) { + errors.addAll(Arrays.asList(list)); + return this; + } + + public ExpectedErrorsBuilder with(Pattern... list) { + regexes.addAll(Arrays.asList(list)); + return this; + } + + public ExpectedErrorsBuilder withRegex(String... list) { + for (String error : list) { + regexes.add(Pattern.compile(error)); + } + return this; + } + + public ExpectedErrors build() { + return new ExpectedErrors(errors, regexes); + } + } } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 5e813dcbd..2f588480f 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -1,5 +1,8 @@ package sqlancer.databend; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class DatabendErrors { @@ -7,7 +10,9 @@ public final class DatabendErrors { private DatabendErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Division by zero"); errors.add("divided by zero"); errors.add("/ by zero"); @@ -45,20 +50,42 @@ public static void addExpressionErrors(ExpectedErrors errors) { * ('067596','19'), ('', '87'); */ errors.add("Can't cast column from null into non-nullable type"); + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Division by zero"); errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); errors.add("number overflowed while evaluating function `"); // 不能在int16类型column上插入int64的数据 + + return errors; } - public static void addGroupByErrors(ExpectedErrors errors) { + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Division by zero"); errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); errors.add("GROUP BY position"); + + return errors; + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); } } diff --git a/src/sqlancer/doris/DorisErrors.java b/src/sqlancer/doris/DorisErrors.java index 678601635..8e6fdfdc9 100644 --- a/src/sqlancer/doris/DorisErrors.java +++ b/src/sqlancer/doris/DorisErrors.java @@ -1,5 +1,8 @@ package sqlancer.doris; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class DorisErrors { @@ -7,7 +10,9 @@ public final class DorisErrors { private DorisErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + // SQL syntax error errors.add("Syntax error"); errors.add("Please check your sql, we meet an error when parsing"); @@ -52,13 +57,27 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (DorisBugs.bug19611) { errors.add("Duplicated inline view column alias"); } + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Insert has filtered data in strict mode"); errors.add("Only value columns of unique table could be updated"); errors.add("Only unique olap table could be updated"); errors.add("Number out of range"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 555bc2e88..5c2909f4d 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.duckdb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class DuckDBErrors { @@ -7,7 +10,9 @@ public final class DuckDBErrors { private DuckDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("with non-constant precision is not supported"); errors.add("Like pattern must not end with escape character"); errors.add("Could not convert string"); @@ -27,9 +32,9 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("GROUP BY clause cannot contain aggregates!"); // investigate - addRegexErrors(errors); + errors.addAll(getRegexErrors()); - addFunctionErrors(errors); + errors.addAll(getFunctionErrors()); errors.add("Overflow in multiplication"); errors.add("Out of Range"); @@ -52,9 +57,17 @@ public static void addExpressionErrors(ExpectedErrors errors) { // timestamp errors.add("Cannot subtract infinite timestamps"); errors.add("Timestamp difference is out of bounds"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); } - private static void addRegexErrors(ExpectedErrors errors) { + private static List getRegexErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("missing ]"); errors.add("missing )"); errors.add("invalid escape sequence"); @@ -64,9 +77,13 @@ private static void addRegexErrors(ExpectedErrors errors) { errors.add("invalid perl operator"); errors.add("invalid character class range"); errors.add("width is not integer"); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("SUBSTRING cannot handle negative lengths"); errors.add("is undefined outside [-1,1]"); // ACOS etc errors.add("invalid type specifier"); // PRINTF @@ -85,11 +102,15 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("Could not choose a best candidate function for the function call"); // monthname errors.add("expected a numeric precision field"); // ROUND errors.add("with non-constant precision is not supported"); // ROUND + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { - addRegexErrors(errors); - addFunctionErrors(errors); + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getRegexErrors()); + errors.addAll(getFunctionErrors()); errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); @@ -105,11 +126,25 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Could not cast value"); errors.add("create unique index, table contains duplicate data"); errors.add("Failed to cast"); + + return errors; } - public static void addGroupByErrors(ExpectedErrors errors) { + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY term out of range"); + + return errors; + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); } } diff --git a/src/sqlancer/h2/H2Errors.java b/src/sqlancer/h2/H2Errors.java index aa1cba9cd..5846c1a88 100644 --- a/src/sqlancer/h2/H2Errors.java +++ b/src/sqlancer/h2/H2Errors.java @@ -1,5 +1,8 @@ package sqlancer.h2; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class H2Errors { @@ -7,7 +10,8 @@ public final class H2Errors { private H2Errors() { } - public static void addInsertErrors(ExpectedErrors errors) { + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); errors.add("NULL not allowed for column"); errors.add("Unique index or primary key violation"); errors.add("Data conversion error"); @@ -16,9 +20,15 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Referential integrity constraint violation"); errors.add("Check constraint invalid"); errors.add("Check constraint violation"); + return errors; } - public static void addExpressionErrors(ExpectedErrors errors) { + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); errors.add("java.lang.ArithmeticException: BigInteger would overflow supported range"); errors.add("Value too long for column"); errors.add("Numeric value out of range"); @@ -41,12 +51,23 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add(/* precision */ "must be between"); // TRUNCATE_VALUE errors.add("Cannot parse \"TIMESTAMP\" constant"); // TRUNCATE errors.add("Invalid parameter count for \"TRUNC\", expected count: \"1\""); // TRUNCATE + return errors; } - public static void addDeleteErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getDeleteErrors() { + ArrayList errors = new ArrayList<>(); errors.add("No default value is set for column"); // referential actions errors.add("Referential integrity constraint violation"); errors.add("NULL not allowed for column"); + return errors; + } + + public static void addDeleteErrors(ExpectedErrors errors) { + errors.addAll(getDeleteErrors()); } } diff --git a/src/sqlancer/hsqldb/HSQLDBErrors.java b/src/sqlancer/hsqldb/HSQLDBErrors.java index 8e2a8cd2f..c02f8f2b9 100644 --- a/src/sqlancer/hsqldb/HSQLDBErrors.java +++ b/src/sqlancer/hsqldb/HSQLDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.hsqldb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class HSQLDBErrors { @@ -7,21 +10,43 @@ public final class HSQLDBErrors { private HSQLDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); errors.add(""); + return errors; } - private static void addRegexErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + private static List getRegexErrors() { + ArrayList errors = new ArrayList<>(); + errors.add(""); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add(""); + + return errors; + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getRegexErrors()); + errors.addAll(getFunctionErrors()); + + return errors; } public static void addInsertErrors(ExpectedErrors errors) { - addRegexErrors(errors); - addFunctionErrors(errors); + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/mariadb/MariaDBErrors.java b/src/sqlancer/mariadb/MariaDBErrors.java index 4918accfb..b84deba02 100644 --- a/src/sqlancer/mariadb/MariaDBErrors.java +++ b/src/sqlancer/mariadb/MariaDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.mariadb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class MariaDBErrors { @@ -7,6 +10,27 @@ public final class MariaDBErrors { private MariaDBErrors() { } + public static List getCommonErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("is out of range"); + // regex + errors.add("unmatched parentheses"); + errors.add("nothing to repeat at offset"); + errors.add("missing )"); + errors.add("missing terminating ]"); + errors.add("range out of order in character class"); + errors.add("unrecognized character after "); + errors.add("Got error '(*VERB) not recognized or malformed"); + errors.add("must be followed by"); + errors.add("malformed number or name after"); + errors.add("digit expected after"); + errors.add("Regex error"); + errors.add("Lock wait timeout exceeded"); + + return errors; + } + public static void addCommonErrors(ExpectedErrors errors) { errors.add("is out of range"); // regex @@ -24,7 +48,8 @@ public static void addCommonErrors(ExpectedErrors errors) { errors.add("Lock wait timeout exceeded"); } - public static void addInsertErrors(ExpectedErrors errors) { + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); errors.add("Out of range"); errors.add("Duplicate entry"); // violates UNIQUE constraint errors.add("cannot be null"); // violates NOT NULL constraint @@ -34,6 +59,11 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("The value specified for generated column"); // trying to insert into a generated column errors.add("Incorrect double value"); errors.add("Incorrect string value"); + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/materialize/gen/MaterializeCommon.java b/src/sqlancer/materialize/gen/MaterializeCommon.java index 7dee348f4..d21c8f81c 100644 --- a/src/sqlancer/materialize/gen/MaterializeCommon.java +++ b/src/sqlancer/materialize/gen/MaterializeCommon.java @@ -1,5 +1,6 @@ package sqlancer.materialize.gen; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.ThreadLocalRandom; import java.util.stream.Collectors; @@ -21,7 +22,9 @@ public final class MaterializeCommon { private MaterializeCommon() { } - public static void addCommonFetchErrors(ExpectedErrors errors) { + public static List getCommonFetchErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); errors.add("but it cannot be referenced from this part of the query"); errors.add("missing FROM-clause entry for table"); @@ -36,14 +39,30 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("does not exist"); errors.add("aggregate functions are not allowed in"); errors.add("is only defined for finite arguments"); + + return errors; } - public static void addCommonTableErrors(ExpectedErrors errors) { + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.addAll(getCommonFetchErrors()); + } + + public static List getCommonTableErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not commutative"); // exclude errors.add("operator requires run-time type coercion"); // exclude + + return errors; } - public static void addCommonExpressionErrors(ExpectedErrors errors) { + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.addAll(getCommonTableErrors()); + } + + public static List getCommonExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); @@ -77,14 +96,22 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("aggregate functions are not allowed in"); errors.add("only defined for finite arguments"); errors.add("unable to parse column reference in GROUP BY clause"); // TODO - addToCharFunctionErrors(errors); - addBitStringOperationErrors(errors); - addFunctionErrors(errors); - addCommonRangeExpressionErrors(errors); - addCommonRegexExpressionErrors(errors); + errors.addAll(getToCharFunctionErrors()); + errors.addAll(getBitStringOperationErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getCommonRangeExpressionErrors()); + errors.addAll(getCommonRegexExpressionErrors()); + + return errors; } - private static void addToCharFunctionErrors(ExpectedErrors errors) { + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + private static List getToCharFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("multiple decimal points"); errors.add("and decimal point together"); errors.add("multiple decimal points"); @@ -96,16 +123,24 @@ private static void addToCharFunctionErrors(ExpectedErrors errors) { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + + return errors; } - private static void addBitStringOperationErrors(ExpectedErrors errors) { + private static List getBitStringOperationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("cannot XOR bit strings of different sizes"); errors.add("cannot AND bit strings of different sizes"); errors.add("cannot OR bit strings of different sizes"); errors.add("must be type boolean, not type text"); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("out of valid range"); // get_bit/get_byte errors.add("cannot take logarithm of a negative number"); errors.add("cannot take logarithm of zero"); @@ -117,24 +152,62 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii errors.add("negative substring length not allowed"); // substr errors.add("invalid mask length"); // set_masklen + + return errors; } - private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + private static List getCommonRegexExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not a valid hexadecimal digit"); + + return errors; } - public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + public static List getCommonRangeExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("range lower bound must be less than or equal to range upper bound"); errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); errors.add("result of range union would not be contiguous"); + + return errors; } - public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + public static List getCommonInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("value too long for type character"); errors.add("not found in view targetlist"); errors.add("CAST does not support casting from"); + + return errors; + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + public static List getGroupingErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("unable to parse column reference in GROUP BY clause"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in"); + + return errors; + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.addAll(getGroupingErrors()); } public static boolean appendDataType(MaterializeDataType type, StringBuilder sb, boolean allowSerial, @@ -332,14 +405,6 @@ private static void deleteOrUpdateAction(StringBuilder sb) { sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); } - public static void addGroupingErrors(ExpectedErrors errors) { - errors.add("non-integer constant in GROUP BY"); // TODO - errors.add("unable to parse column reference in GROUP BY clause"); // TODO - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("is not in select list"); - errors.add("aggregate functions are not allowed in"); - } - public static String getFreeIndexName(MaterializeSchema s) { List indexNames = s.getIndexNames(); String candidateName; diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 9a8f076ce..3f681acb5 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -1,5 +1,8 @@ package sqlancer.mysql; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class MySQLErrors { @@ -7,16 +10,26 @@ public final class MySQLErrors { private MySQLErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) errors.add("is not valid for CHARACTER SET"); if (MySQLBugs.bug111471) { errors.add("Memory capacity exceeded"); } + + return errors; } - public static void addInsertUpdateErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("doesn't have a default value"); errors.add("Data truncation"); errors.add("Incorrect integer value"); @@ -25,6 +38,12 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { errors.add("Data truncated for functional index"); errors.add("cannot be null"); errors.add("Incorrect decimal value"); + + return errors; + } + + public static void addInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getInsertUpdateErrors()); } } diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index 893490c34..822c5497d 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -1,5 +1,8 @@ package sqlancer.oceanbase; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class OceanBaseErrors { @@ -7,7 +10,9 @@ public final class OceanBaseErrors { private OceanBaseErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) errors.add("is not valid for CHARACTER SET"); errors.add("The observer or zone is not the master"); @@ -15,9 +20,17 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Truncated incorrect DOUBLE value"); errors.add("Invalid numeric"); errors.add("Data truncated for argument"); + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Duplicate entry"); errors.add("cannot be null"); errors.add("doesn't have a default value"); @@ -42,5 +55,10 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Invalid numeric"); errors.add("Miss column"); + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 497ad6d0b..a0cc21ae2 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -23,7 +23,9 @@ public final class PostgresCommon { private PostgresCommon() { } - public static void addCommonFetchErrors(ExpectedErrors errors) { + public static List getCommonFetchErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("FULL JOIN is only supported with merge-joinable or hash-joinable join conditions"); errors.add("but it cannot be referenced from this part of the query"); errors.add("missing FROM-clause entry for table"); @@ -33,14 +35,30 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("non-integer constant in GROUP BY"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY position"); + + return errors; } - public static void addCommonTableErrors(ExpectedErrors errors) { + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.addAll(getCommonFetchErrors()); + } + + public static List getCommonTableErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not commutative"); // exclude errors.add("operator requires run-time type coercion"); // exclude + + return errors; } - public static void addCommonExpressionErrors(ExpectedErrors errors) { + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.addAll(getCommonTableErrors()); + } + + public static List getCommonExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); @@ -54,7 +72,6 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("invalid hexadecimal digit"); errors.add("invalid hexadecimal data: odd number of digits"); errors.add("zero raised to a negative power is undefined"); - errors.addRegex(Pattern.compile("cannot convert infinity to \\w+")); errors.add("division by zero"); errors.add("invalid input syntax for type money"); errors.add("invalid input syntax for type"); @@ -65,14 +82,32 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("a negative number raised to a non-integer power yields a complex result"); errors.add("could not determine polymorphic type because input has type unknown"); errors.add("character number must be positive"); - addToCharFunctionErrors(errors); - addBitStringOperationErrors(errors); - addFunctionErrors(errors); - addCommonRangeExpressionErrors(errors); - addCommonRegexExpressionErrors(errors); + errors.addAll(getToCharFunctionErrors()); + errors.addAll(getBitStringOperationErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getCommonRangeExpressionErrors()); + errors.addAll(getCommonRegexExpressionErrors()); + + return errors; + } + + public static List getCommonExpressionRegexErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add(Pattern.compile("cannot convert infinity to \\w+")); + errors.addAll(getFunctionRegexErrors()); + + return errors; + } + + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + errors.addAllRegexes(getCommonExpressionRegexErrors()); } - private static void addToCharFunctionErrors(ExpectedErrors errors) { + private static List getToCharFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("multiple decimal points"); errors.add("and decimal point together"); errors.add("multiple decimal points"); @@ -84,16 +119,24 @@ private static void addToCharFunctionErrors(ExpectedErrors errors) { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + + return errors; } - private static void addBitStringOperationErrors(ExpectedErrors errors) { + private static List getBitStringOperationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("cannot XOR bit strings of different sizes"); errors.add("cannot AND bit strings of different sizes"); errors.add("cannot OR bit strings of different sizes"); errors.add("must be type boolean, not type text"); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("out of valid range"); // get_bit/get_byte errors.add("cannot take logarithm of a negative number"); errors.add("cannot take logarithm of zero"); @@ -103,39 +146,81 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("requested length too large"); // repeat errors.add("invalid memory alloc request size"); // repeat + errors.add("negative substring length not allowed"); // substr + errors.add("invalid mask length"); // set_masklen + + return errors; + } + + private static List getFunctionRegexErrors() { + ArrayList errors = new ArrayList<>(); /* * PostgreSQL support only a few conversion variants to ASCII: LATIN1, LATIN2, LATIN9 and WINDOWS1250. So, it is * better to skip this error at all. */ - errors.addRegex(Pattern.compile("encoding conversion from \\w+ to ASCII not supported")); + errors.add(Pattern.compile("encoding conversion from \\w+ to ASCII not supported")); /* * In accordance with PostgreSQL code, commit 0ab1a2e, conversions to or from SQL_ASCII is meaningless. So * disable errors on such an attempt. */ - errors.addRegex(Pattern.compile("encoding conversion from SQL_ASCII to \\w+ not supported")); - errors.addRegex(Pattern.compile("encoding conversion from \\w+ to SQL_ASCII not supported")); + errors.add(Pattern.compile("encoding conversion from SQL_ASCII to \\w+ not supported")); + errors.add(Pattern.compile("encoding conversion from \\w+ to SQL_ASCII not supported")); - errors.add("negative substring length not allowed"); // substr - errors.add("invalid mask length"); // set_masklen + return errors; } - private static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + private static List getCommonRegexExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not a valid hexadecimal digit"); + + return errors; } - public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + public static List getCommonRangeExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("range lower bound must be less than or equal to range upper bound"); errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); errors.add("result of range union would not be contiguous"); + + return errors; } - public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonRangeExpressionErrors()); + } + + public static List getCommonInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("value too long for type character"); errors.add("cannot insert a non-DEFAULT value into column"); errors.add("not found in view targetlist"); + + return errors; + } + + public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getCommonInsertUpdateErrors()); + } + + public static List getGroupingErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("non-integer constant in GROUP BY"); // TODO + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("is not in select list"); + errors.add("aggregate functions are not allowed in GROUP BY"); + + return errors; + } + + public static void addGroupingErrors(ExpectedErrors errors) { + errors.addAll(getGroupingErrors()); } public static boolean appendDataType(PostgresDataType type, StringBuilder sb, boolean allowSerial, @@ -424,12 +509,4 @@ private static void appendExcludeElement(StringBuilder sb, PostgresGlobalState g private static void deleteOrUpdateAction(StringBuilder sb) { sb.append(Randomly.fromOptions("NO ACTION", "RESTRICT", "CASCADE", "SET NULL", "SET DEFAULT")); } - - public static void addGroupingErrors(ExpectedErrors errors) { - errors.add("non-integer constant in GROUP BY"); // TODO - errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); - errors.add("is not in select list"); - errors.add("aggregate functions are not allowed in GROUP BY"); - } - } diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 603addb5c..68a99ef9f 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -1,5 +1,8 @@ package sqlancer.presto; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class PrestoErrors { @@ -7,7 +10,9 @@ public final class PrestoErrors { private PrestoErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + // Presto errors errors.add("cannot be applied to"); errors.add("LIKE expression must evaluate to a varchar"); @@ -57,9 +62,17 @@ public static void addExpressionErrors(ExpectedErrors errors) { // ARRAY errors.add("Unknown type: ARRAY"); + + return errors; } - private static void addRegexErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + private static List getRegexErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("missing ]"); errors.add("missing )"); errors.add("invalid escape sequence"); @@ -69,9 +82,13 @@ private static void addRegexErrors(ExpectedErrors errors) { errors.add("invalid perl operator"); errors.add("invalid character class range"); errors.add("width is not integer"); + + return errors; } - private static void addFunctionErrors(ExpectedErrors errors) { + private static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("SUBSTRING cannot handle negative lengths"); errors.add("is undefined outside [-1,1]"); // ACOS etc errors.add("invalid type specifier"); // PRINTF @@ -90,12 +107,16 @@ private static void addFunctionErrors(ExpectedErrors errors) { errors.add("Could not choose a best candidate function for the function call"); // monthname errors.add("expected a numeric precision field"); // ROUND errors.add("with non-constant precision is not supported"); // ROUND + + return errors; } // TODO: cover presto error - public static void addInsertErrors(ExpectedErrors errors) { - addRegexErrors(errors); - addFunctionErrors(errors); + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.addAll(getRegexErrors()); + errors.addAll(getFunctionErrors()); errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); @@ -118,10 +139,23 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Values rows have mismatched types"); errors.add("Invalid numeric literal"); + return errors; } - public static void addGroupByErrors(ExpectedErrors errors) { + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } + + public static List getGroupByErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("must be an aggregate expression or appear in GROUP BY clause"); + + return errors; + } + + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); } } diff --git a/src/sqlancer/questdb/QuestDBErrors.java b/src/sqlancer/questdb/QuestDBErrors.java index 83c74fdc1..213f06ad4 100644 --- a/src/sqlancer/questdb/QuestDBErrors.java +++ b/src/sqlancer/questdb/QuestDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.questdb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class QuestDBErrors { @@ -7,7 +10,9 @@ public final class QuestDBErrors { private QuestDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + // TODO (anxing) errors.add("unexpected argument for function: "); errors.add("unexpected token:"); // SELECT FROM multiple tables without WHERE/ JOIN clause @@ -16,16 +21,36 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("too few arguments for 'in'"); errors.add("cannot compare TIMESTAMP with type"); // WHERE column IN with nonTIMESTAMP arg errors.add("constant expected"); + + return errors; } - public static void addGroupByErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getGroupByErrors() { // TODO (anxing) + + return new ArrayList<>(); } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addGroupByErrors(ExpectedErrors errors) { + errors.addAll(getGroupByErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + // TODO (anxing) errors.add("Invalid column"); errors.add("inconvertible types:"); errors.add("inconvertible value:"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 6827116af..406729bfb 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -1,6 +1,8 @@ package sqlancer.sqlite3; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import sqlancer.common.query.ExpectedErrors; @@ -9,14 +11,24 @@ public final class SQLite3Errors { private SQLite3Errors() { } - public static void addDeleteErrors(ExpectedErrors errors) { + public static List getDeleteErrors() { + ArrayList errors = new ArrayList<>(); + // DELETE trigger for a view/table to which colomns were added or deleted errors.add("columns but"); // trigger with on conflict clause errors.add("ON CONFLICT clause does not match any PRIMARY KEY or UNIQUE constraint"); + + return errors; } - public static void addExpectedExpressionErrors(ExpectedErrors errors) { + public static void addDeleteErrors(ExpectedErrors errors) { + errors.addAll(getDeleteErrors()); + } + + public static List getExpectedExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("[SQLITE_BUSY] The database file is locked"); errors.add("FTS expression tree is too large"); errors.add("String or BLOB exceeds size limit"); @@ -67,9 +79,16 @@ public static void addExpectedExpressionErrors(ExpectedErrors errors) { errors.add("circularly defined"); errors.add("[SQLITE_ERROR] SQL error or missing database"); // A possible delay in the execution of DROP TABLE // statement. + return errors; } - public static void addMatchQueryErrors(ExpectedErrors errors) { + public static void addExpectedExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpectedExpressionErrors()); + } + + public static List getMatchQueryErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("unable to use function MATCH in the requested context"); errors.add("malformed MATCH expression"); errors.add("fts5: syntax error near"); @@ -78,28 +97,60 @@ public static void addMatchQueryErrors(ExpectedErrors errors) { errors.add("fts5: column queries are not supported"); // vt0.c0 MATCH '2016456922' errors.add("fts5: phrase queries are not supported"); errors.add("unterminated string"); + + return errors; } - public static void addTableManipulationErrors(ExpectedErrors errors) { + public static void addMatchQueryErrors(ExpectedErrors errors) { + errors.addAll(getMatchQueryErrors()); + } + + public static List getTableManipulationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("unsupported frame specification"); errors.add("non-deterministic functions prohibited in CHECK constraints"); errors.addAll(Arrays.asList("subqueries prohibited in CHECK constraints", "generated columns cannot be part of the PRIMARY KEY", "must have at least one non-generated column")); + + return errors; } - public static void addQueryErrors(ExpectedErrors errors) { + public static void addTableManipulationErrors(ExpectedErrors errors) { + errors.addAll(getTableManipulationErrors()); + } + + public static List getQueryErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("ON clause references tables to its right"); + + return errors; } - public static void addInsertNowErrors(ExpectedErrors errors) { + public static void addQueryErrors(ExpectedErrors errors) { + errors.addAll(getQueryErrors()); + } + + public static List getInsertNowErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("non-deterministic use of strftime()"); errors.add("non-deterministic use of time()"); errors.add("non-deterministic use of datetime()"); errors.add("non-deterministic use of julianday()"); errors.add("non-deterministic use of date()"); + + return errors; } - public static void addInsertUpdateErrors(ExpectedErrors errors) { + public static void addInsertNowErrors(ExpectedErrors errors) { + errors.addAll(getInsertNowErrors()); + } + + public static List getInsertUpdateErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("String or BLOB exceeds size limit"); errors.add("[SQLITE_CONSTRAINT_CHECK]"); errors.add("[SQLITE_CONSTRAINT_PRIMARYKEY]"); @@ -114,6 +165,12 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { errors.add("[SQLITE_ERROR] SQL error or missing database (no such table:"); errors.add("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch"); errors.add("no such column"); // trigger + + return errors; + } + + public static void addInsertUpdateErrors(ExpectedErrors errors) { + errors.addAll(getInsertUpdateErrors()); } } diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java index 152c5bfc9..d8b0fd416 100644 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ b/src/sqlancer/stonedb/StoneDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.stonedb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; @@ -7,36 +10,54 @@ public final class StoneDBErrors { private StoneDBErrors() { } - public static void addExpectedExpressionErrors(StoneDBGlobalState globalState, ExpectedErrors errors) { + public static List getExpectedExpressionErrors(StoneDBGlobalState globalState) { + ArrayList errors = new ArrayList<>(); + // java.sql.SQLException: Incorrect DATE value: '292269055-12-02' errors.add("Incorrect DATE value: "); - // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 + // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for + // column 'c1' at row 1 errors.add("Incorrect string value: "); // java.sql.SQLException: Incorrect integer value: 'ST' for column 'c1' at row 1 errors.add("Incorrect integer value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Truncated incorrect INTEGER value: '#Q' + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Truncated + // incorrect INTEGER value: '#Q' errors.add("Data truncation: Truncated incorrect INTEGER value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT value is out of range in + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT + // value is out of range in // '-((`database0`.`t0`.`c1` >> (not(`database0`.`t0`.`c1`))))' errors.add("Data truncation: BIGINT value is out of range in "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT UNSIGNED value is out of range in + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT + // UNSIGNED value is out of range in // '(`database10`.`t0`.`c0` + (`database10`.`t0`.`c0` & (not(0.5))))' errors.add("Data truncation: BIGINT UNSIGNED value is out of range in "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect time value: '0Sly6xqF0' for + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect + // time value: '0Sly6xqF0' for // column 'c1' at row 1 errors.add("Data truncation: Incorrect time value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: DOUBLE value is out of range in + // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: DOUBLE + // value is out of range in // '(`database0`.`t0`.`c0` * `database0`.`t0`.`c0`)' errors.add("Data truncation: DOUBLE value is out of range in "); - // java.sql.SQLException: Numeric result of an expression is too large and cannot be handled by tianmu. + // java.sql.SQLException: Numeric result of an expression is too large and + // cannot be handled by tianmu. errors.add("Numeric result of an expression is too large and cannot be handled by tianmu."); - // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order clause' + // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order + // clause' errors.add("Unknown column "); if (globalState.getDbmsSpecificOptions().test80Version) { - // Caused by: java.sql.SQLException: Incorrect DATETIME value: '292269055-12-02 16:47:04' + // Caused by: java.sql.SQLException: Incorrect DATETIME value: '292269055-12-02 + // 16:47:04' errors.add("Incorrect DATETIME value: "); - // Caused by: java.sql.SQLException: Incorrect TIMESTAMP value: '292269055-12-02 16:47:04' + // Caused by: java.sql.SQLException: Incorrect TIMESTAMP value: '292269055-12-02 + // 16:47:04' errors.add("Incorrect TIMESTAMP value: "); } + + return errors; + } + + public static void addExpectedExpressionErrors(StoneDBGlobalState globalState, ExpectedErrors errors) { + errors.addAll(getExpectedExpressionErrors(globalState)); } } diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index fd4d23d3d..f10587a8b 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -1,5 +1,8 @@ package sqlancer.tidb; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class TiDBErrors { @@ -7,7 +10,9 @@ public final class TiDBErrors { private TiDBErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("DECIMAL value is out of range"); errors.add("error parsing regexp"); errors.add("BIGINT UNSIGNED value is out of range"); @@ -58,14 +63,30 @@ public static void addExpressionErrors(ExpectedErrors errors) { if (TiDBBugs.bug44747) { errors.add("index out of range"); } + + return errors; } - public static void addExpressionHavingErrors(ExpectedErrors errors) { + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getExpressionHavingErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not in GROUP BY clause and contains nonaggregated column"); errors.add("Unknown column"); + + return errors; } - public static void addInsertErrors(ExpectedErrors errors) { + public static void addExpressionHavingErrors(ExpectedErrors errors) { + errors.addAll(getExpressionHavingErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Duplicate entry"); errors.add("cannot be null"); errors.add("doesn't have a default value"); @@ -84,6 +105,12 @@ public static void addInsertErrors(ExpectedErrors errors) { errors.add("Incorrect decimal value"); errors.add("error parsing regexp"); errors.add("is not valid for CHARACTER SET"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); } } diff --git a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java index 1a13b9cb4..69c4148de 100644 --- a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java +++ b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java @@ -1,5 +1,8 @@ package sqlancer.timescaledb.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class TimescaleDBCommon { @@ -7,7 +10,11 @@ private TimescaleDBCommon() { } - public static void addTimescaleDBErrors(ExpectedErrors errors) { + public static List getTimescaleDBErrors() { + return new ArrayList<>(); + } + public static void addTimescaleDBErrors(ExpectedErrors errors) { + errors.addAll(getTimescaleDBErrors()); } } diff --git a/src/sqlancer/yugabyte/ycql/YCQLErrors.java b/src/sqlancer/yugabyte/ycql/YCQLErrors.java index 2bcddb71c..010b2111a 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLErrors.java +++ b/src/sqlancer/yugabyte/ycql/YCQLErrors.java @@ -1,5 +1,8 @@ package sqlancer.yugabyte.ycql; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class YCQLErrors { @@ -7,7 +10,9 @@ public final class YCQLErrors { private YCQLErrors() { } - public static void addExpressionErrors(ExpectedErrors errors) { + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("Signature mismatch in call to builtin function"); errors.add("Qualified name not allowed for column reference"); errors.add("Datatype Mismatch"); @@ -16,6 +21,12 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.add("Invalid SQL Statement"); errors.add("Order by clause contains invalid expression"); errors.add("Invalid Function Call"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); } } diff --git a/src/sqlancer/yugabyte/ysql/YSQLErrors.java b/src/sqlancer/yugabyte/ysql/YSQLErrors.java index a2cee1add..de692b352 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLErrors.java +++ b/src/sqlancer/yugabyte/ysql/YSQLErrors.java @@ -1,5 +1,8 @@ package sqlancer.yugabyte.ysql; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class YSQLErrors { @@ -7,7 +10,9 @@ public final class YSQLErrors { private YSQLErrors() { } - public static void addCommonFetchErrors(ExpectedErrors errors) { + public static List getCommonFetchErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("An I/O error occurred while sending to the backend"); errors.add("Conflicts with committed transaction"); errors.add("cannot be changed"); @@ -22,9 +27,17 @@ public static void addCommonFetchErrors(ExpectedErrors errors) { errors.add("non-integer constant in"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY position"); + + return errors; } - public static void addCommonTableErrors(ExpectedErrors errors) { + public static void addCommonFetchErrors(ExpectedErrors errors) { + errors.addAll(getCommonFetchErrors()); + } + + public static List getCommonTableErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("PRIMARY KEY containing column of type 'INET' not yet supported"); errors.add("PRIMARY KEY containing column of type 'VARBIT' not yet supported"); errors.add("PRIMARY KEY containing column of type 'INT4RANGE' not yet supported"); @@ -34,9 +47,17 @@ public static void addCommonTableErrors(ExpectedErrors errors) { errors.add("is not commutative"); // exclude errors.add("cannot be changed"); errors.add("operator requires run-time type coercion"); // exclude + + return errors; } - public static void addCommonExpressionErrors(ExpectedErrors errors) { + public static void addCommonTableErrors(ExpectedErrors errors) { + errors.addAll(getCommonTableErrors()); + } + + public static List getCommonExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("syntax error at or near \"(\""); errors.add("does not exist"); errors.add("is not unique"); @@ -69,14 +90,22 @@ public static void addCommonExpressionErrors(ExpectedErrors errors) { errors.add("a negative number raised to a non-integer power yields a complex result"); errors.add("could not determine polymorphic type because input has type unknown"); - addToCharFunctionErrors(errors); - addBitStringOperationErrors(errors); - addFunctionErrors(errors); - addCommonRangeExpressionErrors(errors); - addCommonRegexExpressionErrors(errors); + errors.addAll(getToCharFunctionErrors()); + errors.addAll(getBitStringOperationErrors()); + errors.addAll(getFunctionErrors()); + errors.addAll(getCommonRangeExpressionErrors()); + errors.addAll(getCommonRegexExpressionErrors()); + + return errors; } - public static void addToCharFunctionErrors(ExpectedErrors errors) { + public static void addCommonExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonExpressionErrors()); + } + + public static List getToCharFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("multiple decimal points"); errors.add("and decimal point together"); errors.add("multiple decimal points"); @@ -88,16 +117,32 @@ public static void addToCharFunctionErrors(ExpectedErrors errors) { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + + return errors; } - public static void addBitStringOperationErrors(ExpectedErrors errors) { + public static void addToCharFunctionErrors(ExpectedErrors errors) { + errors.addAll(getToCharFunctionErrors()); + } + + public static List getBitStringOperationErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("cannot XOR bit strings of different sizes"); errors.add("cannot AND bit strings of different sizes"); errors.add("cannot OR bit strings of different sizes"); errors.add("must be type boolean, not type text"); + + return errors; } - public static void addFunctionErrors(ExpectedErrors errors) { + public static void addBitStringOperationErrors(ExpectedErrors errors) { + errors.addAll(getBitStringOperationErrors()); + } + + public static List getFunctionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("out of valid range"); // get_bit/get_byte errors.add("cannot take logarithm of a negative number"); errors.add("cannot take logarithm of zero"); @@ -109,18 +154,40 @@ public static void addFunctionErrors(ExpectedErrors errors) { errors.add("encoding conversion from UTF8 to ASCII not supported"); // to_ascii errors.add("negative substring length not allowed"); // substr errors.add("invalid mask length"); // set_masklen + + return errors; } - public static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + public static void addFunctionErrors(ExpectedErrors errors) { + errors.addAll(getFunctionErrors()); + } + + public static List getCommonRegexExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("is not a valid hexadecimal digit"); + + return errors; } - public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + public static void addCommonRegexExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonRangeExpressionErrors()); + } + + public static List getCommonRangeExpressionErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("range lower bound must be less than or equal to range upper bound"); errors.add("result of range difference would not be contiguous"); errors.add("out of range"); errors.add("malformed range literal"); errors.add("result of range union would not be contiguous"); + + return errors; + } + + public static void addCommonRangeExpressionErrors(ExpectedErrors errors) { + errors.addAll(getCommonRangeExpressionErrors()); } public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { @@ -128,14 +195,24 @@ public static void addCommonInsertUpdateErrors(ExpectedErrors errors) { errors.add("not found in view targetlist"); } - public static void addGroupingErrors(ExpectedErrors errors) { + public static List getGroupingErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("non-integer constant in GROUP BY"); // TODO errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("is not in select list"); errors.add("aggregate functions are not allowed in GROUP BY"); + + return errors; } - public static void addViewErrors(ExpectedErrors errors) { + public static void addGroupingErrors(ExpectedErrors errors) { + errors.addAll(getGroupingErrors()); + } + + public static List getViewErrors() { + ArrayList errors = new ArrayList<>(); + errors.add("already exists"); errors.add("cannot drop columns from view"); errors.add("non-integer constant in ORDER BY"); // TODO @@ -147,5 +224,11 @@ public static void addViewErrors(ExpectedErrors errors) { errors.add("is not a view"); errors.add("non-integer constant in DISTINCT ON"); errors.add("SELECT DISTINCT ON expressions must match initial ORDER BY expressions"); + + return errors; + } + + public static void addViewErrors(ExpectedErrors errors) { + errors.addAll(getViewErrors()); } } diff --git a/test/sqlancer/TestExpectedErrors.java b/test/sqlancer/TestExpectedErrors.java index 2781df89b..2cb938824 100644 --- a/test/sqlancer/TestExpectedErrors.java +++ b/test/sqlancer/TestExpectedErrors.java @@ -1,5 +1,6 @@ package sqlancer; +import java.util.List; import java.util.regex.Pattern; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -31,6 +32,18 @@ public void testStringSimple() { } + @Test + public void testStringsSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addAll(List.of("a", "b", "c")); + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + } + @Test public void testRegexSimple() { ExpectedErrors errors = new ExpectedErrors(); @@ -44,6 +57,41 @@ public void testRegexSimple() { } + @Test + public void testRegexesSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addAllRegexes(List.of(Pattern.compile("a\\d"), Pattern.compile("b\\D"))); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } + + @Test + public void testRegexStringSimple() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addRegexString("a\\d"); + errors.addRegexString("b\\D"); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + } + + @Test + public void testRegexStrings() { + ExpectedErrors errors = new ExpectedErrors(); + errors.addAllRegexStrings(List.of("a\\d", "b\\D")); + errors.add("c"); + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } + @Test public void testStringRealistic() { ExpectedErrors errors = new ExpectedErrors(); @@ -60,4 +108,28 @@ public void testRegexRealistic() { assertTrue(errors.errorIsExpected("PRIMARY KEY constraint was violated!")); } + @Test + public void testBuilder() { + ExpectedErrors errors = ExpectedErrors.newErrors().with("a", "b", "c").build(); + + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + errors = ExpectedErrors.newErrors().withRegex("a\\d", "b\\D").with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + errors = ExpectedErrors.newErrors().with(Pattern.compile("a\\d"), Pattern.compile("b\\D")).with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + } } From 8cd8246e4cadc3e5e1c30524563424420b5c3313 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 25 Mar 2024 12:06:18 +0800 Subject: [PATCH 413/856] Rename `xxxExpressions()` to `xxxClauses()` for select --- src/sqlancer/clickhouse/ClickHouseToStringVisitor.java | 4 ++-- src/sqlancer/clickhouse/ast/ClickHouseSelect.java | 4 ++-- .../oracle/tlp/ClickHouseTLPAggregateOracle.java | 4 ++-- .../clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java | 4 ++-- src/sqlancer/cnosdb/CnosDBToStringVisitor.java | 4 ++-- src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java | 2 +- .../cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java | 2 +- src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java | 2 +- src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java | 2 +- src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java | 4 ++-- src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java | 4 ++-- .../gen/CockroachDBRandomQuerySynthesizer.java | 2 +- .../cockroachdb/oracle/CockroachDBNoRECOracle.java | 2 +- .../oracle/tlp/CockroachDBTLPAggregateOracle.java | 8 +++++--- .../oracle/tlp/CockroachDBTLPExtendedWhereOracle.java | 2 +- .../oracle/tlp/CockroachDBTLPHavingOracle.java | 2 +- .../cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java | 2 +- src/sqlancer/common/ast/SelectBase.java | 4 ++-- src/sqlancer/common/ast/newast/Select.java | 4 ++-- src/sqlancer/databend/DatabendToStringVisitor.java | 4 ++-- .../databend/gen/DatabendRandomQuerySynthesizer.java | 5 +++-- src/sqlancer/databend/test/DatabendNoRECOracle.java | 4 ++-- .../test/DatabendPivotedQuerySynthesisOracle.java | 2 +- .../tlp/DatabendQueryPartitioningAggregateTester.java | 2 +- .../test/tlp/DatabendQueryPartitioningHavingTester.java | 4 ++-- .../test/tlp/DatabendQueryPartitioningWhereTester.java | 2 +- src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java | 2 +- src/sqlancer/doris/oracle/DorisNoRECOracle.java | 2 +- .../doris/oracle/DorisPivotedQuerySynthesisOracle.java | 2 +- .../oracle/tlp/DorisQueryPartitioningAggregateTester.java | 2 +- .../oracle/tlp/DorisQueryPartitioningHavingTester.java | 2 +- .../oracle/tlp/DorisQueryPartitioningWhereTester.java | 2 +- src/sqlancer/doris/visitor/DorisToStringVisitor.java | 4 ++-- src/sqlancer/duckdb/DuckDBToStringVisitor.java | 4 ++-- src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java | 2 +- src/sqlancer/duckdb/test/DuckDBNoRECOracle.java | 2 +- .../test/DuckDBQueryPartitioningAggregateTester.java | 2 +- .../duckdb/test/DuckDBQueryPartitioningHavingTester.java | 2 +- .../duckdb/test/DuckDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/h2/H2QueryPartitioningWhereTester.java | 2 +- src/sqlancer/h2/H2RandomQuerySynthesizer.java | 2 +- src/sqlancer/h2/H2ToStringVisitor.java | 4 ++-- src/sqlancer/hsqldb/HSQLDBToStringVisitor.java | 4 ++-- src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java | 2 +- .../hsqldb/test/HSQLDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/materialize/MaterializeToStringVisitor.java | 4 ++-- .../materialize/gen/MaterializeRandomQueryGenerator.java | 2 +- .../materialize/oracle/MaterializeNoRECOracle.java | 3 +-- .../oracle/MaterializePivotedQuerySynthesisOracle.java | 2 +- .../oracle/tlp/MaterializeTLPAggregateOracle.java | 2 +- .../materialize/oracle/tlp/MaterializeTLPBase.java | 2 +- .../oracle/tlp/MaterializeTLPHavingOracle.java | 5 +++-- .../materialize/oracle/tlp/MaterializeTLPWhereOracle.java | 4 ++-- src/sqlancer/mysql/MySQLToStringVisitor.java | 6 +++--- src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java | 2 +- .../mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 2 +- src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java | 4 ++-- src/sqlancer/oceanbase/OceanBaseToStringVisitor.java | 6 +++--- .../oracle/OceanBasePivotedQuerySynthesisOracle.java | 2 +- .../oceanbase/oracle/OceanBaseTLPWhereOracle.java | 4 ++-- src/sqlancer/postgres/PostgresToStringVisitor.java | 4 ++-- .../postgres/gen/PostgresRandomQueryGenerator.java | 2 +- src/sqlancer/postgres/oracle/PostgresNoRECOracle.java | 2 +- .../oracle/PostgresPivotedQuerySynthesisOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPAggregateOracle.java | 2 +- src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java | 2 +- .../postgres/oracle/tlp/PostgresTLPHavingOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPWhereOracle.java | 4 ++-- src/sqlancer/presto/PrestoToStringVisitor.java | 4 ++-- src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java | 5 +++-- .../presto/gen/PrestoTypedExpressionGenerator.java | 5 +++-- src/sqlancer/presto/test/PrestoNoRECOracle.java | 3 +-- .../test/PrestoQueryPartitioningAggregateTester.java | 2 +- .../presto/test/PrestoQueryPartitioningHavingTester.java | 2 +- .../presto/test/PrestoQueryPartitioningWhereTester.java | 2 +- src/sqlancer/questdb/QuestDBToStringVisitor.java | 4 ++-- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 4 ++-- src/sqlancer/sqlite3/ast/SQLite3Select.java | 4 ++-- src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java | 2 +- .../oracle/SQLite3PivotedQuerySynthesisOracle.java | 2 +- .../sqlite3/oracle/SQLite3RandomQuerySynthesizer.java | 5 +++-- .../sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java | 4 ++-- .../sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java | 2 +- src/sqlancer/stonedb/StoneDBToStringVisitor.java | 4 ++-- src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java | 2 +- src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java | 2 +- .../oracle/StoneDBQueryPartitioningHavingTester.java | 5 +++-- .../oracle/StoneDBQueryPartitioningWhereTester.java | 2 +- src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java | 2 +- src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 5 +++-- src/sqlancer/tidb/oracle/TiDBDQPOracle.java | 2 +- src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java | 2 +- src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java | 2 +- src/sqlancer/tidb/visitor/TiDBToStringVisitor.java | 4 ++-- src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java | 4 ++-- .../yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java | 2 +- src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java | 4 ++-- .../yugabyte/ysql/gen/YSQLRandomQueryGenerator.java | 2 +- src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java | 2 +- .../ysql/oracle/YSQLPivotedQuerySynthesisOracle.java | 2 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java | 2 +- src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 2 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java | 2 +- .../yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java | 4 ++-- 106 files changed, 160 insertions(+), 153 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index 37f4d3ab6..cebce8fc2 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -97,9 +97,9 @@ public void visit(ClickHouseSelect select, boolean inner) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByClause().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByClause()); + visit(select.getOrderByClauses()); } if (inner) { sb.append(")"); diff --git a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java index 24d489982..3904b767b 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java @@ -64,11 +64,11 @@ public ClickHouseExpression getLimitClause() { return limitClause; } - public List getOrderByClause() { + public List getOrderByClauses() { return orderByClause; } - public void setOrderByExpressions(List orderBy) { + public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java index d14356a57..cedac9c51 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPAggregateOracle.java @@ -25,7 +25,7 @@ public ClickHouseTLPAggregateOracle(ClickHouseProvider.ClickHouseGlobalState sta public void check() throws SQLException { super.check(); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } @@ -49,7 +49,7 @@ public void check() throws SQLException { .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java index 92195bedf..be2bd71fc 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java @@ -23,7 +23,7 @@ public ClickHouseTLPWhereOracle(ClickHouseProvider.ClickHouseGlobalState state) public void check() throws SQLException { super.check(); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } String originalQueryString = ClickHouseVisitor.asString(select); @@ -31,7 +31,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(IntStream.range(0, 1 + Randomly.smallNumber()) + select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); } diff --git a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java index 64f2fd8b8..44d3bf509 100644 --- a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java +++ b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java @@ -146,9 +146,9 @@ public void visit(CnosDBSelect s) { visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByExpressions()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java index df91e9a58..0c817c655 100644 --- a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java +++ b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java @@ -141,7 +141,7 @@ private int getOptimizedQueryCount(List randomTables, List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, CnosDBExpectedError.expectedErrors(), state); - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = CnosDBVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java index 1c6fe0f36..67abfdbbe 100644 --- a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java @@ -102,9 +102,9 @@ public void visit(CockroachDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java index d3679b015..09f5b165d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java @@ -56,7 +56,7 @@ public static CockroachDBSelect generateSelect(CockroachDBGlobalState globalStat select.setWhereClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.getOrderingTerms()); + select.setOrderByClauses(gen.getOrderingTerms()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index c627b9151..2bc1873a0 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -89,7 +89,7 @@ private int getOptimizedResult(CockroachDBExpression whereCondition, List mapped(CockroachDBAggregate aggregate) { case MIN: return aliasArgs(Arrays.asList(aggregate)); case AVG: - // List arg = Arrays.asList(new CockroachDBCast(aggregate.getExpr().get(0), + // List arg = Arrays.asList(new + // CockroachDBCast(aggregate.getExpr().get(0), // CockroachDBDataType.DECIMAL.get())); CockroachDBAggregate sum = new CockroachDBAggregate(CockroachDBAggregateFunction.SUM, aggregate.getExpr()); CockroachDBCast count = new CockroachDBCast( new CockroachDBAggregate(CockroachDBAggregateFunction.COUNT, aggregate.getExpr()), CockroachDBDataType.DECIMAL.get()); - // CockroachDBBinaryArithmeticOperation avg = new CockroachDBBinaryArithmeticOperation(sum, count, + // CockroachDBBinaryArithmeticOperation avg = new + // CockroachDBBinaryArithmeticOperation(sum, count, // CockroachDBBinaryArithmeticOperator.DIV); return aliasArgs(Arrays.asList(sum, count)); default: diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java index 7463e767f..863f056ab 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPExtendedWhereOracle.java @@ -38,7 +38,7 @@ public void check() throws SQLException { boolean allowOrderBy = Randomly.getBoolean(); if (allowOrderBy) { - select.setOrderByExpressions(gen.getOrderingTerms()); + select.setOrderByClauses(gen.getOrderingTerms()); } select.setWhereClause(combinePredicate(predicate)); String firstQueryString = CockroachDBVisitor.asString(select); diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java index d5a96d80b..7a7dd7779 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPHavingOracle.java @@ -28,7 +28,7 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java index db0ea6162..0fe935d18 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java @@ -32,7 +32,7 @@ public void check() throws SQLException { boolean allowOrderBy = Randomly.getBoolean(); if (allowOrderBy) { - select.setOrderByExpressions(gen.getOrderingTerms()); + select.setOrderByClauses(gen.getOrderingTerms()); } CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); select.setWhereClause(predicate); diff --git a/src/sqlancer/common/ast/SelectBase.java b/src/sqlancer/common/ast/SelectBase.java index 9e02a7f61..ebc3ed5c3 100644 --- a/src/sqlancer/common/ast/SelectBase.java +++ b/src/sqlancer/common/ast/SelectBase.java @@ -59,14 +59,14 @@ public List getGroupByExpressions() { return groupByExpressions; } - public void setOrderByExpressions(List orderByExpressions) { + public void setOrderByClauses(List orderByExpressions) { if (orderByExpressions == null) { throw new IllegalArgumentException(); } this.orderByExpressions = orderByExpressions; } - public List getOrderByExpressions() { + public List getOrderByClauses() { assert orderByExpressions != null; return orderByExpressions; } diff --git a/src/sqlancer/common/ast/newast/Select.java b/src/sqlancer/common/ast/newast/Select.java index 0953f289a..53520648d 100644 --- a/src/sqlancer/common/ast/newast/Select.java +++ b/src/sqlancer/common/ast/newast/Select.java @@ -24,9 +24,9 @@ public interface Select, E extends Expression, T exte Expression getLimitClause(); - List getOrderByClause(); + List getOrderByClauses(); - void setOrderByExpressions(List orderBy); + void setOrderByClauses(List orderBy); void setOffsetClause(E offsetClause); diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java index b3669e4b7..87c94d074 100644 --- a/src/sqlancer/databend/DatabendToStringVisitor.java +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -77,9 +77,9 @@ private void visit(DatabendSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index cd2427c2f..4f0b2fd00 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -64,8 +64,9 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int List> noExprColumns = new ArrayList<>(columnOfLeafNode); if (Randomly.getBoolean() && !noExprColumns.isEmpty() && !isDistinct) { - select.setOrderByExpressions(Randomly.nonEmptySubset(noExprColumns)); - // TODO (for SELECT DISTINCT, ORDER BY expressions must appear in select list) isDistinct + select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); + // TODO (for SELECT DISTINCT, ORDER BY expressions must appear in select list) + // isDistinct // 需要orderby输入每个select list，可以用数字代替比如：1,2,3... } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index d8cf09a4a..50a8c4de4 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -113,8 +113,8 @@ private int getOptimizedQueryCount(SQLConnection con, List getRectifiedQuery() throws Exception { DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); if (!isDistinct) { List> orderBys = gen.generateOrderBy(); - selectStatement.setOrderByExpressions(orderBys); + selectStatement.setOrderByClauses(orderBys); } return new SQLQueryAdapter(DatabendToStringVisitor.asString(selectStatement), errors); } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index b8463bd53..043e93963 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -58,7 +58,7 @@ public void check() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); // if (Randomly.getBooleanWithRatherLowProbability()) { - // select.setOrderByExpressions(gen.generateOrderBys()); + // select.setOrderByClauses(gen.generateOrderBys()); // } originalQuery = DatabendToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java index 2eda63ab5..dfde7755f 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java @@ -33,11 +33,11 @@ public void check() throws SQLException { // boolean orderBy = Randomly.getBoolean(); boolean orderBy = false; // 关闭order by if (orderBy) { // TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select - // select.setOrderByExpressions(gen.generateOrderBys()); + // select.setOrderByClauses(gen.generateOrderBys()); List> constants = new ArrayList<>(); constants.add(new DatabendConstant.DatabendIntConstant( Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } select.setGroupByExpressions(groupByExpression); select.setHavingClause(null); diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index a9ac6a6c3..d6fb7f0e6 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -28,7 +28,7 @@ public void check() throws SQLException { // boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); boolean orderBy = false; // if (orderBy) { //TODO 待开启 - // select.setOrderByExpressions(gen.generateOrderBys()); + // select.setOrderByClauses(gen.generateOrderBys()); // } select.setWhereClause(DatabendExprToNode.cast(predicate)); String firstQueryString = DatabendToStringVisitor.asString(select); diff --git a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java index 3193594df..1b85d4b34 100644 --- a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java +++ b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java @@ -62,7 +62,7 @@ public static DorisSelect generateSelect(DorisGlobalState globalState, int nrCol List> noExprColumns = new ArrayList<>(columnOfLeafNode); if (Randomly.getBoolean()) { - select.setOrderByExpressions(Randomly.nonEmptySubset(noExprColumns)); + select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); } if (Randomly.getBoolean()) { select.setGroupByExpressions(noExprColumns); diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java index dab6b53ae..01fa11f2f 100644 --- a/src/sqlancer/doris/oracle/DorisNoRECOracle.java +++ b/src/sqlancer/doris/oracle/DorisNoRECOracle.java @@ -114,7 +114,7 @@ private int getOptimizedQueryCount(SQLConnection con, List List> constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } select.setJoinList(joins); int firstCount = 0; diff --git a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java index 95c79393b..4d8f3c012 100644 --- a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java @@ -71,7 +71,7 @@ protected Query getRectifiedQuery() throws Exception { List> constants = new ArrayList<>(); constants.add(new DorisConstant.DorisIntConstant( Randomly.smallNumber() % selectStatement.getFetchColumns().size() + 1)); - selectStatement.setOrderByExpressions(constants); + selectStatement.setOrderByClauses(constants); } return new SQLQueryAdapter(DorisToStringVisitor.asString(selectStatement), errors); } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java index 2e886afbe..d8076d3e5 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -65,7 +65,7 @@ public void check() throws SQLException { List> constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } originalQuery = DorisToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java index 2151e1da3..676065616 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -36,7 +36,7 @@ public void check() throws SQLException { List> constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } select.setGroupByExpressions(groupByExpression); select.setHavingClause(null); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java index e373e98b9..34ade1d3a 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -35,7 +35,7 @@ public void check() throws SQLException { List> constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByExpressions(constants); + select.setOrderByClauses(constants); } select.setWhereClause(DorisExprToNode.cast(predicate)); String firstQueryString = DorisToStringVisitor.asString(select); diff --git a/src/sqlancer/doris/visitor/DorisToStringVisitor.java b/src/sqlancer/doris/visitor/DorisToStringVisitor.java index 846955c83..f7770058f 100644 --- a/src/sqlancer/doris/visitor/DorisToStringVisitor.java +++ b/src/sqlancer/doris/visitor/DorisToStringVisitor.java @@ -147,9 +147,9 @@ private void visit(DorisSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index f7c97f3e5..37493d0f6 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -68,9 +68,9 @@ private void visit(DuckDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java index 29af42e10..aef88ef13 100644 --- a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java +++ b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java @@ -48,7 +48,7 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC select.setWhereClause(gen.generateExpression()); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index de89bfa7d..2ce6a8ad2 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -113,7 +113,7 @@ private int getFirstQueryCount(SQLConnection con, List> t select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions(new DuckDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); + select.setOrderByClauses(new DuckDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); } // select.setSelectType(SelectType.ALL); select.setJoinList(joins); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index 8d7835d29..de8bd96a0 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -58,7 +58,7 @@ public void check() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = DuckDBToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index e8988d867..b75bf02a5 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -30,7 +30,7 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java index ca9cde670..7e20e605c 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java @@ -27,7 +27,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = DuckDBToStringVisitor.asString(select); diff --git a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java index 8de4ecd37..cc09dea7a 100644 --- a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java +++ b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java @@ -24,7 +24,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = H2ToStringVisitor.asString(select); diff --git a/src/sqlancer/h2/H2RandomQuerySynthesizer.java b/src/sqlancer/h2/H2RandomQuerySynthesizer.java index 37942f0e5..dafa7835c 100644 --- a/src/sqlancer/h2/H2RandomQuerySynthesizer.java +++ b/src/sqlancer/h2/H2RandomQuerySynthesizer.java @@ -36,7 +36,7 @@ public static H2Select generateSelect(H2GlobalState globalState, int nrColumns) select.setWhereClause(gen.generateExpression()); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/h2/H2ToStringVisitor.java b/src/sqlancer/h2/H2ToStringVisitor.java index cb86ede30..ad45c66f2 100644 --- a/src/sqlancer/h2/H2ToStringVisitor.java +++ b/src/sqlancer/h2/H2ToStringVisitor.java @@ -67,9 +67,9 @@ public void visit(H2Select select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java index 722e241fa..8caf7b867 100644 --- a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java +++ b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java @@ -74,9 +74,9 @@ private void visit(HSQLDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java index ca376e348..eb08d061d 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -101,7 +101,7 @@ private int getFirstQueryCount(SQLConnection con, List> t select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions(new HSQLDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); + select.setOrderByClauses(new HSQLDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); int firstCount = 0; diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java index 0b1706583..2b793383e 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java @@ -23,7 +23,7 @@ public void check() throws Exception { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(expressionGenerator.generateOrderBys()); + select.setOrderByClauses(expressionGenerator.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = HSQLDBToStringVisitor.asString(select); diff --git a/src/sqlancer/materialize/MaterializeToStringVisitor.java b/src/sqlancer/materialize/MaterializeToStringVisitor.java index 6a3f06445..b39179cae 100644 --- a/src/sqlancer/materialize/MaterializeToStringVisitor.java +++ b/src/sqlancer/materialize/MaterializeToStringVisitor.java @@ -148,9 +148,9 @@ public void visit(MaterializeSelect s) { visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByExpressions()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java index 16f4f1be8..c8b1441fc 100644 --- a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java @@ -46,7 +46,7 @@ public static MaterializeSelect createRandomQuery(int nrColumns, MaterializeGlob } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java index fbfbcb132..bf9cc43fd 100644 --- a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java +++ b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java @@ -147,8 +147,7 @@ private int getOptimizedQueryCount(List randomTables, Lis select.setFromList(randomTables); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( - new MaterializeExpressionGenerator(state).setColumns(columns).generateOrderBy()); + select.setOrderByClauses(new MaterializeExpressionGenerator(state).setColumns(columns).generateOrderBy()); } select.setSelectType(SelectType.ALL); select.setJoinClauses(joinStatements); diff --git a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java index 07bfb470f..f4b2a1cc6 100644 --- a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java +++ b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java @@ -64,7 +64,7 @@ public SQLQueryAdapter getRectifiedQuery() throws SQLException { } List orderBy = new MaterializeExpressionGenerator(globalState).setColumns(columns) .generateOrderBy(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(MaterializeVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java index 4f5f67294..e8154bb9a 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java @@ -63,7 +63,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } originalQuery = MaterializeVisitor.asString(select); generatedQueryString = originalQuery; diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java index 4aa2196d1..dcf16dfad 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java @@ -106,7 +106,7 @@ public static MaterializeSubquery createSubquery(MaterializeGlobalState globalSt select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java index 46d6c6d74..e4328ce40 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java @@ -36,7 +36,8 @@ protected void havingCheck() throws SQLException { generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - // See https://github.com/MaterializeInc/materialize/issues/18346, have to check if predicate errors by putting + // See https://github.com/MaterializeInc/materialize/issues/18346, have to check + // if predicate errors by putting // it in SELECT first List originalColumns = select.getFetchColumns(); List checkColumns = new ArrayList<>(); @@ -48,7 +49,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } select.setHavingClause(predicate); String firstQueryString = MaterializeVisitor.asString(select); diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java index 6af0ac2cc..6cec373a9 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java @@ -25,13 +25,13 @@ public void check() throws SQLException { protected void whereCheck() throws SQLException { if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } String originalQueryString = MaterializeVisitor.asString(select); generatedQueryString = originalQueryString; List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = MaterializeVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 757232406..c4bce3559 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -101,14 +101,14 @@ public void visit(MySQLSelect s) { visit(groupBys.get(i)); } } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - List orderBys = s.getOrderByExpressions(); + List orderBys = s.getOrderByClauses(); for (int i = 0; i < orderBys.size(); i++) { if (i != 0) { sb.append(", "); } - visit(s.getOrderByExpressions().get(i)); + visit(s.getOrderByClauses().get(i)); } } if (s.getLimitClause() != null) { diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java index 8ebaff3b6..e1a7da75f 100644 --- a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -33,7 +33,7 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) select.setWhereClause(gen.generateExpression()); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 6e50fbb33..28665c328 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -68,7 +68,7 @@ public Query getRectifiedQuery() throws SQLException { selectStatement.setModifiers(modifiers); List orderBy = new MySQLExpressionGenerator(globalState).setColumns(columns) .generateOrderBys(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(MySQLVisitor.asString(selectStatement), errors); } diff --git a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java index 4b578a24d..3d4de3873 100644 --- a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java @@ -25,9 +25,9 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = MySQLVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java index 56fef98f1..a65ae8ee3 100644 --- a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java @@ -100,14 +100,14 @@ public void visit(OceanBaseSelect s) { sb.append(" HAVING "); visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - List orderBys = s.getOrderByExpressions(); + List orderBys = s.getOrderByClauses(); for (int i = 0; i < orderBys.size(); i++) { if (i != 0) { sb.append(", "); } - visit(s.getOrderByExpressions().get(i)); + visit(s.getOrderByClauses().get(i)); } } if (s.getLimitClause() != null) { diff --git a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java index 48fe1002e..8fa0a4bc0 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBasePivotedQuerySynthesisOracle.java @@ -71,7 +71,7 @@ public Query getRectifiedQuery() throws SQLException { selectStatement.setOffsetClause(offsetClause); } List orderBy = generateOrderBy(columns); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(OceanBaseVisitor.asString(selectStatement), errors); } diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java index 73a0c3a6f..7c8c64e3c 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java @@ -25,9 +25,9 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = OceanBaseVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index d1766e6c8..17327ab9c 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -170,9 +170,9 @@ public void visit(PostgresSelect s) { visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByExpressions()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java index d1527ca8b..7fc5049cd 100644 --- a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java @@ -45,7 +45,7 @@ public static PostgresSelect createRandomQuery(int nrColumns, PostgresGlobalStat } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index f4bc10e74..2ad7cc2b7 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -144,7 +144,7 @@ private int getOptimizedQueryCount(List randomTables, List

orderBy = new PostgresExpressionGenerator(globalState).setColumns(columns) .generateOrderBy(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(PostgresVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index a1eaa43c7..a1ae5597e 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -61,7 +61,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } originalQuery = PostgresVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 5e349b5dc..2e34b1f87 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -103,7 +103,7 @@ public static PostgresSubquery createSubquery(PostgresGlobalState globalState, S select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java index 3a1275299..6af683903 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java @@ -36,7 +36,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } select.setHavingClause(predicate); String firstQueryString = PostgresVisitor.asString(select); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java index b043f6981..d8aeb4938 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java @@ -24,12 +24,12 @@ public void check() throws SQLException { protected void whereCheck() throws SQLException { if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } String originalQueryString = PostgresVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = PostgresVisitor.asString(select); select.setWhereClause(negatedPredicate); diff --git a/src/sqlancer/presto/PrestoToStringVisitor.java b/src/sqlancer/presto/PrestoToStringVisitor.java index 1342ff635..f840cf850 100644 --- a/src/sqlancer/presto/PrestoToStringVisitor.java +++ b/src/sqlancer/presto/PrestoToStringVisitor.java @@ -99,9 +99,9 @@ private void visit(PrestoSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java index 167c1f222..2d18311b6 100644 --- a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java +++ b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java @@ -50,7 +50,7 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC select.setWhereClause(gen.generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull())); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); @@ -61,7 +61,8 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC } // if (Randomly.getBoolean()) { // select.setOffsetClause( - // PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + // PrestoConstant.createIntConstant(Randomly.getNotCachedInteger(0, + // Integer.MAX_VALUE))); // } if (Randomly.getBoolean()) { select.setHavingClause(gen.generateHavingClause()); diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java index 7afb8da29..79e657fda 100644 --- a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -342,7 +342,7 @@ private PrestoSelect generateSubquery(List columns) { Node predicate = typedExpressionGenerator.generatePredicate(); select.setWhereClause(predicate); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions(typedExpressionGenerator.generateOrderBys()); + select.setOrderByClauses(typedExpressionGenerator.generateOrderBys()); } List> joins = PrestoJoin.getJoins(tableList, globalState); select.setJoinList(joins); @@ -741,7 +741,8 @@ public static BinaryOperatorNode.Operator getRandomForType(PrestoSchema.PrestoCo case ARRAY: case INTERVAL_YEAR_TO_MONTH: case INTERVAL_DAY_TO_SECOND: - // return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, IS_DISTINCT_FROM, + // return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, + // IS_DISTINCT_FROM, // IS_NOT_DISTINCT_FROM); default: return Randomly.fromOptions(EQUALS, NOT_EQUALS, NOT_EQUALS_ALT, IS_DISTINCT_FROM, IS_NOT_DISTINCT_FROM); diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java index a63d912b0..da09b5a01 100644 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -114,8 +114,7 @@ private int getFirstQueryCount(SQLConnection con, List> t select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( - new PrestoTypedExpressionGenerator(state).setColumns(columns).generateOrderBys()); + select.setOrderByClauses(new PrestoTypedExpressionGenerator(state).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); int firstCount = 0; diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java index 527d5d648..0f10a22d4 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java @@ -54,7 +54,7 @@ public void check() throws SQLException { aggregateFunction); select.setFetchColumns(List.of(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = PrestoToStringVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java index b3ff3beaa..ad11806bf 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java @@ -31,7 +31,7 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java index 093c27108..cfff823c5 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java @@ -28,7 +28,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = PrestoToStringVisitor.asString(select); diff --git a/src/sqlancer/questdb/QuestDBToStringVisitor.java b/src/sqlancer/questdb/QuestDBToStringVisitor.java index 295a6d910..7b2c889d2 100644 --- a/src/sqlancer/questdb/QuestDBToStringVisitor.java +++ b/src/sqlancer/questdb/QuestDBToStringVisitor.java @@ -49,9 +49,9 @@ private void visit(QuestDBSelect select) { // sb.append(" HAVING "); // visit(select.getHavingClause()); // } - // if (!select.getOrderByExpressions().isEmpty()) { + // if (!select.getOrderByClauses().isEmpty()) { // sb.append(" ORDER BY "); - // visit(select.getOrderByExpressions()); + // visit(select.getOrderByClauses()); // } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 475c16335..b8b860943 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -148,9 +148,9 @@ public void visit(SQLite3Select s, boolean inner) { sb.append(" HAVING "); visit(s.getHavingClause()); } - if (!s.getOrderByClause().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByClause()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 5ff008709..36beb5fd8 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -86,11 +86,11 @@ public SQLite3Expression getLimitClause() { return limitClause; } - public List getOrderByClause() { + public List getOrderByClauses() { return orderByClause; } - public void setOrderByExpressions(List orderBy) { + public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 45ec6631d..2ec2e392f 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -127,7 +127,7 @@ private Function getOptimizedQuery(SQLite3Select se SQLite3Expression randomWhereCondition) throws SQLException { boolean useAggregate = Randomly.getBoolean(); if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (useAggregate) { select.setFetchColumns(Arrays.asList(new SQLite3Aggregate(Collections.emptyList(), diff --git a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java index 4021c88e9..98a30dc23 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3PivotedQuerySynthesisOracle.java @@ -99,7 +99,7 @@ public SQLite3Select getQuery() throws SQLException { } /* PQS does not check for ordering, so we can generate any ORDER BY clause */ List orderBy = new SQLite3ExpressionGenerator(globalState).generateOrderBys(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); if (!groupByClause.isEmpty() && Randomly.getBoolean()) { selectStatement.setHavingClause(generateRectifiedExpression(columns, pivotRow, true)); } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java index 4447ea2fb..d3d7ded31 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java @@ -102,7 +102,8 @@ public static SQLite3Expression generate(SQLite3GlobalState globalState, int siz select.setFromList(SQLite3Common.getTableRefs(tables, s)); // TODO: no values are referenced from this sub query yet // if (Randomly.getBooleanWithSmallProbability()) { - // select.getFromList().add(SQLite3RandomQuerySynthesizer.generate(globalState, Randomly.smallNumber() + 1)); + // select.getFromList().add(SQLite3RandomQuerySynthesizer.generate(globalState, + // Randomly.smallNumber() + 1)); // } // WHERE @@ -121,7 +122,7 @@ public static SQLite3Expression generate(SQLite3GlobalState globalState, int siz boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { // ORDER BY - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBooleanWithRatherLowProbability()) { // LIMIT diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java index 849a8640f..4d1dd8529 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPAggregateOracle.java @@ -54,7 +54,7 @@ public void check() throws SQLException { List from = SQLite3Common.getTableRefs(targetTables.getTables(), s); select.setFromList(from); if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } String originalQuery = SQLite3Visitor.asString(select); generatedQueryString = originalQuery; @@ -120,7 +120,7 @@ private SQLite3Select getSelect(SQLite3Aggregate aggregate, List(joins)); select.setFromList(new ArrayList<>(tableReferenceNodeList)); select.setWhereClause(gen.generateExpression()); - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); } diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java index 2465494ef..eb1482d7f 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java @@ -47,7 +47,7 @@ public void check() throws Exception { select.setFetchColumns(fetchColumns); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } String originalQuery = StoneDBToStringVisitor.asString(select); diff --git a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java index 6cd5b3bc9..ebf1f9967 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java @@ -55,7 +55,7 @@ public void check() throws Exception { select.setJoinList(joins); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByExpressions( + select.setOrderByClauses( new StoneDBExpressionGenerator(globalState).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java index 744d52f55..d6efe30d1 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java @@ -112,7 +112,7 @@ private int getOptimizedQueryCount(SQLConnection con, List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - // specific to combined query string, will produce the same result as original query string in logic + // specific to combined query string, will produce the same result as original + // query string in logic select.setHavingClause(predicate); String firstQueryString = StoneDBToStringVisitor.asString(select); select.setHavingClause(negatedPredicate); diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java index 2a8f2b4d4..4c9267f68 100644 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java @@ -23,7 +23,7 @@ public void check() throws Exception { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = StoneDBToStringVisitor.asString(select); diff --git a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java index f0609ad56..1b1599ac2 100644 --- a/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java +++ b/src/sqlancer/tidb/gen/TiDBRandomQuerySynthesizer.java @@ -41,7 +41,7 @@ public static TiDBSelect generateSelect(TiDBGlobalState globalState, int nrColum select.setWhereClause(gen.generateExpression()); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java index 4e56dd5fa..4fd5c8c54 100644 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -63,7 +63,7 @@ public void check() throws SQLException { select.setWhereClause(gen.generateExpression()); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(select.getFetchColumns()); @@ -115,7 +115,8 @@ protected boolean mutateJoin() { TiDBJoin join = (TiDBJoin) Randomly.fromList(select.getJoinList()); // CROSS does not need ON Condition, while other joins do - // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + // To avoid Null pointer, generating a new new condition when mutating CROSS to + // other joins if (join.getJoinType() == JoinType.CROSS) { List columns = new ArrayList<>(); columns.addAll(((TiDBTableReference) join.getLeftTable()).getTable().getColumns()); diff --git a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java index e35e4ee70..17b288b7d 100644 --- a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java @@ -54,7 +54,7 @@ public void check() throws SQLException { select.setWhereClause(gen.generateExpression()); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(gen.generateExpression()); diff --git a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java index 23b3cb035..76e1a3c46 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPHavingOracle.java @@ -29,7 +29,7 @@ public void check() throws SQLException { } boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); select.setHavingClause(null); diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java index f3640800e..e437b2856 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java @@ -66,7 +66,7 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setWhereClause(predicate); String firstQueryString = TiDBVisitor.asString(select); diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index e9c199407..67c7cda31 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -74,9 +74,9 @@ public void visit(TiDBSelect select) { sb.append(" HAVING "); visit(select.getHavingClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } } diff --git a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java index ddc7e6e8a..59adf3e6b 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java +++ b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java @@ -41,9 +41,9 @@ private void visit(YCQLSelect select) { sb.append(" WHERE "); visit(select.getWhereClause()); } - if (!select.getOrderByExpressions().isEmpty()) { + if (!select.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(select.getOrderByExpressions()); + visit(select.getOrderByClauses()); } if (select.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java index 7a10009ad..05f9ec2dd 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java @@ -38,7 +38,7 @@ public static YCQLSelect generateSelect(YCQLGlobalState globalState, int nrColum select.setWhereClause(gen.generateExpression()); } if (Randomly.getBoolean()) { - select.setOrderByExpressions(gen.generateOrderBys()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setGroupByExpressions(Randomly.nonEmptySubset(select.getFetchColumns())); diff --git a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java index 39732b728..eef4c7d9f 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java +++ b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java @@ -136,9 +136,9 @@ public void visit(YSQLSelect s) { visit(s.getHavingClause()); } - if (!s.getOrderByExpressions().isEmpty()) { + if (!s.getOrderByClauses().isEmpty()) { sb.append(" ORDER BY "); - visit(s.getOrderByExpressions()); + visit(s.getOrderByClauses()); } if (s.getLimitClause() != null) { sb.append(" LIMIT "); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java index 03109227d..02e87dffd 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java @@ -45,7 +45,7 @@ public static YSQLSelect createRandomQuery(int nrColumns, YSQLGlobalState global } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java index 3ae27fd73..37a6491ad 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -141,7 +141,7 @@ private int getOptimizedQueryCount(List randomTables, List orderBy = new YSQLExpressionGenerator(globalState).setColumns(columns).generateOrderBy(); - selectStatement.setOrderByExpressions(orderBy); + selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(YSQLVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java index 537a074ae..6e9b74f87 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -61,7 +61,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } originalQuery = YSQLVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java index 863e85968..fa63ffc1f 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -53,7 +53,7 @@ public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } if (Randomly.getBoolean()) { select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java index 2302143c6..d4a398cf2 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java @@ -43,7 +43,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } select.setHavingClause(predicate); String firstQueryString = YSQLVisitor.asString(select); diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java index 0af8095d0..0d61dd8c5 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java @@ -24,12 +24,12 @@ public void check() throws SQLException { protected void whereCheck() throws SQLException { if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByExpressions(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBy()); } String originalQueryString = YSQLVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setOrderByExpressions(Collections.emptyList()); + select.setOrderByClauses(Collections.emptyList()); select.setWhereClause(predicate); String firstQueryString = YSQLVisitor.asString(select); select.setWhereClause(negatedPredicate); From d8f3899167b7b9683b638f2b115a2edfb1edd14e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 26 Mar 2024 17:41:21 +0800 Subject: [PATCH 414/856] Downgrade DuckDB from 5.1 to 4.0 --- .github/workflows/main.yml | 2 ++ pom.xml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af92103d7..b063ce6d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -7,12 +7,14 @@ on: paths: - 'src/**' - '.github/workflows/*.yml' + - 'pom.xml' pull_request: branches: - main paths: - 'src/**' - '.github/workflows/*.yml' + - 'pom.xml' jobs: misc: diff --git a/pom.xml b/pom.xml index 292378933..2b34d6c27 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 0.5.1 + 0.4.0 com.facebook.presto From b68bc6e6f788b21b3b383fecb0c7fba9bf24f10d Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 26 Mar 2024 17:42:07 +0800 Subject: [PATCH 415/856] Include NoREC in DuckDB unit tests --- .github/workflows/main.yml | 4 +++- test/sqlancer/dbms/TestDuckDBNoREC.java | 16 ++++++++++++++++ .../dbms/{TestDuckDB.java => TestDuckDBTLP.java} | 4 ++-- 3 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 test/sqlancer/dbms/TestDuckDBNoREC.java rename test/sqlancer/dbms/{TestDuckDB.java => TestDuckDBTLP.java} (90%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b063ce6d8..71425b0e2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -214,7 +214,9 @@ jobs: - name: Build run: mvn -B package -DskipTests=true - name: DuckDB Tests - run: mvn -Dtest=TestDuckDB test + run: | + mvn -Dtest=TestDuckDBTLP test + mvn -Dtest=TestDuckDBNoREC test h2: name: DBMS Tests (H2) diff --git a/test/sqlancer/dbms/TestDuckDBNoREC.java b/test/sqlancer/dbms/TestDuckDBNoREC.java new file mode 100644 index 000000000..dea1c70dd --- /dev/null +++ b/test/sqlancer/dbms/TestDuckDBNoREC.java @@ -0,0 +1,16 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestDuckDBNoREC { + @Test + public void testDuckDBNoREC() { + // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "duckdb", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestDuckDB.java b/test/sqlancer/dbms/TestDuckDBTLP.java similarity index 90% rename from test/sqlancer/dbms/TestDuckDB.java rename to test/sqlancer/dbms/TestDuckDBTLP.java index d36d8610f..322e3eeb4 100644 --- a/test/sqlancer/dbms/TestDuckDB.java +++ b/test/sqlancer/dbms/TestDuckDBTLP.java @@ -6,10 +6,10 @@ import sqlancer.Main; -public class TestDuckDB { +public class TestDuckDBTLP { @Test - public void testDuckDB() { + public void testDuckDBTLP() { // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, From a5b25f700c24e60f7f7ee408475613005a9be9f8 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Tue, 26 Mar 2024 23:01:39 +0800 Subject: [PATCH 416/856] Update SQLite3 tests (#917) * Update SQLite3 tests * Update CI to run when tests/ is updated --- .github/workflows/main.yml | 5 ++++- ...{TestSQLite3.java => TestSQLiteNoREC.java} | 7 +++---- test/sqlancer/dbms/TestSQLiteTLP.java | 19 +++++++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) rename test/sqlancer/dbms/{TestSQLite3.java => TestSQLiteNoREC.java} (53%) create mode 100644 test/sqlancer/dbms/TestSQLiteTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af92103d7..25016aa96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -6,12 +6,14 @@ on: - main paths: - 'src/**' + - 'test/**' - '.github/workflows/*.yml' pull_request: branches: - main paths: - 'src/**' + - 'test/**' - '.github/workflows/*.yml' jobs: @@ -383,7 +385,8 @@ jobs: - name: SQLite Tests run: | mvn -Dtest=TestSQLitePQS test - mvn -Dtest=TestSQLite3 test + mvn -Dtest=TestSQLiteTLP test + mvn -Dtest=TestSQLiteNoREC test sqlite-qpg: name: QPG Tests (SQLite) diff --git a/test/sqlancer/dbms/TestSQLite3.java b/test/sqlancer/dbms/TestSQLiteNoREC.java similarity index 53% rename from test/sqlancer/dbms/TestSQLite3.java rename to test/sqlancer/dbms/TestSQLiteNoREC.java index f26b449c1..aa7741659 100644 --- a/test/sqlancer/dbms/TestSQLite3.java +++ b/test/sqlancer/dbms/TestSQLiteNoREC.java @@ -6,13 +6,12 @@ import sqlancer.Main; -public class TestSQLite3 { +public class TestSQLiteNoREC { @Test - public void testSqlite() { - // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 + public void testSqliteNoREC() { assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", "0", "sqlite3" })); + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", "NoREC" })); } } diff --git a/test/sqlancer/dbms/TestSQLiteTLP.java b/test/sqlancer/dbms/TestSQLiteTLP.java new file mode 100644 index 000000000..dfeb83193 --- /dev/null +++ b/test/sqlancer/dbms/TestSQLiteTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLiteTLP { + + @Test + public void testSqliteTLP() { + // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 + assertEquals(0, + Main.executeMain(new String[] {"--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", + "QUERY_PARTITIONING"})); + } +} From 82bb0c30510bf1ef7e4fcb6215fda5a9d9add2b0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 26 Mar 2024 23:53:14 +0800 Subject: [PATCH 417/856] Fix a formatting error (#930) --- test/sqlancer/dbms/TestSQLiteTLP.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sqlancer/dbms/TestSQLiteTLP.java b/test/sqlancer/dbms/TestSQLiteTLP.java index dfeb83193..7d90fd93c 100644 --- a/test/sqlancer/dbms/TestSQLiteTLP.java +++ b/test/sqlancer/dbms/TestSQLiteTLP.java @@ -12,8 +12,8 @@ public class TestSQLiteTLP { public void testSqliteTLP() { // run with one thread due to multithreading issues, see https://github.com/sqlancer/sqlancer/pull/45 assertEquals(0, - Main.executeMain(new String[] {"--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", - "QUERY_PARTITIONING"})); + "QUERY_PARTITIONING" })); } } From 4e33b7e1e2b69c880fa122b34938b1d28149767b Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Wed, 27 Mar 2024 11:24:44 +0800 Subject: [PATCH 418/856] [WIP] Generic NoREC oracle (#928) * Add NoRECGenerator interface * Add `toSum` method for Select interface Used in the NoREC oracle to aggregate unoptimized query results * Add `getRandomTableNonEmptyTables` function * Add generic NoREC oracle * Check for null arguments in NoREC oracle constructor * Update NoREC oracle implementation and add comments for generator --- src/sqlancer/common/gen/NoRECGenerator.java | 49 ++++++ src/sqlancer/common/oracle/NoRECOracle.java | 141 ++++++++++++++++++ .../common/oracle/TestOracleUtils.java | 22 +++ 3 files changed, 212 insertions(+) create mode 100644 src/sqlancer/common/gen/NoRECGenerator.java create mode 100644 src/sqlancer/common/oracle/NoRECOracle.java create mode 100644 src/sqlancer/common/oracle/TestOracleUtils.java diff --git a/src/sqlancer/common/gen/NoRECGenerator.java b/src/sqlancer/common/gen/NoRECGenerator.java new file mode 100644 index 000000000..f57f7c20f --- /dev/null +++ b/src/sqlancer/common/gen/NoRECGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface NoRECGenerator, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + NoRECGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + Select generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + /** + * Generates a query string that is likely to be optimized by the DBMS. + * + * @param select + * the base select expression used to generate the query + * @param whereCondition + * a condition where records will be checked with + * @param shouldUseAggregate + * whether to aggregate the record counts (`true`) or display records as is (`false`) + * + * @return a query string to be executed + */ + String generateOptimizedQueryString(Select select, E whereCondition, boolean shouldUseAggregate); + + /** + * Generates a query string that is unlikely to be optimized by the DBMS. + * + * @param select + * the base select expression used to generate the query + * @param whereCondition + * the condition each record will be checked with + * + * @return a query string to be executed + */ + String generateUnoptimizedQueryString(Select select, E whereCondition); +} diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java new file mode 100644 index 000000000..1fbad671a --- /dev/null +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -0,0 +1,141 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class NoRECOracle, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + + private NoRECGenerator gen; + private final ExpectedErrors errors; + + private String lastQueryString; + + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + Select select = gen.generateSelect(); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + + E randomWhereCondition = gen.generateBooleanExpression(); + + boolean shouldUseAggregate = Randomly.getBoolean(); + String optimizedQueryString = gen.generateOptimizedQueryString(select, randomWhereCondition, + shouldUseAggregate); + lastQueryString = optimizedQueryString; + + String unoptimizedQueryString = gen.generateUnoptimizedQueryString(select, randomWhereCondition); + + int optimizedCount = shouldUseAggregate ? extractCounts(optimizedQueryString, errors, state) + : countRows(optimizedQueryString, errors, state); + int unoptimizedCount = extractCounts(optimizedQueryString, errors, state); + + if (optimizedCount == -1 || unoptimizedCount == -1) { + throw new IgnoreMeException(); + } + + if (unoptimizedCount != optimizedCount) { + String queryFormatString = "-- %s;\n-- count: %d"; + String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); + String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, + unoptimizedCount); + state.getState().getLocalState() + .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); + String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", optimizedCount, + unoptimizedCount, firstQueryStringWithCount, secondQueryStringWithCount); + throw new AssertionError(assertionMessage); + } + } + + @Override + public String getLastQueryString() { + return lastQueryString; + } + + private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(queryString); + } + + int count = 0; + try (SQLancerResultSet rs = q.executeAndGet(state)) { + if (rs == null) { + return -1; + } else { + try { + while (rs.next()) { + count++; + } + } catch (SQLException e) { + count = -1; + } + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw (IgnoreMeException) e; + } + throw new AssertionError(q.getQueryString(), e); + } + return count; + } + + private int extractCounts(String queryString, ExpectedErrors errors, SQLGlobalState state) { + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(queryString); + } + + int count = 0; + try (SQLancerResultSet rs = q.executeAndGet(state)) { + if (rs == null) { + return -1; + } else { + try { + while (rs.next()) { + count += rs.getInt(1); + } + } catch (SQLException e) { + count = -1; + } + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw (IgnoreMeException) e; + } + throw new AssertionError(q.getQueryString(), e); + } + return count; + } + +} diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java new file mode 100644 index 000000000..b22cc89f5 --- /dev/null +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -0,0 +1,22 @@ +package sqlancer.common.oracle; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public final class TestOracleUtils { + + private TestOracleUtils() { + } + + public static , C extends AbstractTableColumn> AbstractTables getRandomTableNonEmptyTables( + AbstractSchema schema) { + if (schema.getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables())); + } +} From 0c6dcbd3a3f209cb1f52f5c5013b7a45547a7c2a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 27 Mar 2024 11:47:52 +0800 Subject: [PATCH 419/856] Remove the TiDB incremental ANALYZE feature (#931) This feature currently fails the CI: "the incremental analyze feature has already been removed in TiDB v7.5.0, so this will have no effect." --- src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 8bcea30fb..f78878ac0 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -17,11 +17,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx ExpectedErrors errors = new ExpectedErrors(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); - StringBuilder sb = new StringBuilder("ANALYZE "); - if (analyzeIndex && Randomly.getBoolean()) { - sb.append("INCREMENTAL "); - } - sb.append("TABLE "); + StringBuilder sb = new StringBuilder("ANALYZE TABLE "); sb.append(table.getName()); if (analyzeIndex) { sb.append(" INDEX "); From 419038a7f634d136feffb2c8e0ea599ac5fce7da Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 27 Mar 2024 11:10:21 +0800 Subject: [PATCH 420/856] Add expected insert errors for TiDB v7.5.1 --- src/sqlancer/tidb/TiDBErrors.java | 3 +++ src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java | 1 + 2 files changed, 4 insertions(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index f10587a8b..4f45debce 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -29,6 +29,7 @@ public static List getExpressionErrors() { errors.add("doesn't have a default value"); // default errors.add("is not valid for CHARACTER SET"); errors.add("DOUBLE value is out of range"); + errors.add("Result of space() was larger than max_allowed_packet"); errors.add("Data truncat"); errors.add("Truncated incorrect FLOAT value"); @@ -105,6 +106,8 @@ public static List getInsertErrors() { errors.add("Incorrect decimal value"); errors.add("error parsing regexp"); errors.add("is not valid for CHARACTER SET"); + errors.add("for function inet_aton"); + errors.add("'Empty pattern is invalid' from regexp"); return errors; } diff --git a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java index 535cf973f..4d0807d4b 100644 --- a/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAlterTableGenerator.java @@ -33,6 +33,7 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { errors.add("A PRIMARY must include all columns in the table's partitioning function"); errors.add("key was too long"); errors.add("Duplicate entry"); + errors.add("has a partitioning function dependency and cannot be dropped or renamed"); StringBuilder sb = new StringBuilder("ALTER TABLE "); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBColumn column = table.getRandomColumn(); From 1c47e40725ec598bd0753abe86981b8a26e25d7f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 27 Mar 2024 11:11:04 +0800 Subject: [PATCH 421/856] Pin TiDB version in CI to v7.5.1 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f618ab1ca..786593077 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -459,7 +459,7 @@ jobs: - name: Set up TiDB run: | docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v7.5.1 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" From 56c5e98ca20da7bf91f28553b4d8b694237cacd8 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Thu, 28 Mar 2024 08:15:15 +0800 Subject: [PATCH 422/856] [SQLite3] Use generic NoREC implementation (#933) * Implement Join interface for SQlite3 Join * Implement Select interface for SQLite3 Select * Fix typo in NoREC oracle * Add logging to NoREC oracle * Add reproducer to NoREC oracle * Implement NoRECGenerator interface for SQLite3 * Use generic NoREC oracle for SQLite3 --- src/sqlancer/common/ast/newast/Join.java | 2 - src/sqlancer/common/oracle/NoRECOracle.java | 41 +++- .../sqlite3/ast/SQLite3Expression.java | 9 +- src/sqlancer/sqlite3/ast/SQLite3Select.java | 30 ++- .../gen/SQLite3ExpressionGenerator.java | 77 +++++++- .../sqlite3/oracle/SQLite3NoRECOracle.java | 179 ++---------------- 6 files changed, 168 insertions(+), 170 deletions(-) diff --git a/src/sqlancer/common/ast/newast/Join.java b/src/sqlancer/common/ast/newast/Join.java index b1d1ed280..90935b338 100644 --- a/src/sqlancer/common/ast/newast/Join.java +++ b/src/sqlancer/common/ast/newast/Join.java @@ -6,8 +6,6 @@ public interface Join, T extends AbstractTable, C extends AbstractTableColumn> extends Expression { - T getTable(); - Expression getOnClause(); void setOnClause(E onClause); diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 1fbad671a..aac047b2c 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -1,9 +1,12 @@ package sqlancer.common.oracle; import java.sql.SQLException; +import java.util.Objects; +import java.util.function.Function; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.SQLGlobalState; import sqlancer.common.ast.newast.Expression; import sqlancer.common.ast.newast.Join; @@ -25,8 +28,24 @@ public class NoRECOracle, E extends Expression, S ext private NoRECGenerator gen; private final ExpectedErrors errors; + private Reproducer reproducer; private String lastQueryString; + private static class NoRECReproducer> implements Reproducer { + private final Function optimizedQuery; + private final Function unoptimizedQuery; + + NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery) { + this.optimizedQuery = optimizedQuery; + this.unoptimizedQuery = unoptimizedQuery; + } + + @Override + public boolean bugStillTriggers(G globalState) { + return !Objects.equals(optimizedQuery.apply(globalState), unoptimizedQuery.apply(globalState)); + } + } + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -34,10 +53,12 @@ public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expec this.state = state; this.gen = gen; this.errors = expectedErrors; + this.reproducer = null; } @Override public void check() throws SQLException { + reproducer = null; S schema = state.getSchema(); AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); gen = gen.setTablesAndColumns(targetTables); @@ -52,18 +73,31 @@ public void check() throws SQLException { String optimizedQueryString = gen.generateOptimizedQueryString(select, randomWhereCondition, shouldUseAggregate); lastQueryString = optimizedQueryString; + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(optimizedQueryString); + } String unoptimizedQueryString = gen.generateUnoptimizedQueryString(select, randomWhereCondition); + if (state.getOptions().logEachSelect()) { + state.getLogger().writeCurrent(unoptimizedQueryString); + } int optimizedCount = shouldUseAggregate ? extractCounts(optimizedQueryString, errors, state) : countRows(optimizedQueryString, errors, state); - int unoptimizedCount = extractCounts(optimizedQueryString, errors, state); + int unoptimizedCount = extractCounts(unoptimizedQueryString, errors, state); if (optimizedCount == -1 || unoptimizedCount == -1) { throw new IgnoreMeException(); } if (unoptimizedCount != optimizedCount) { + Function optimizedQuery = state -> shouldUseAggregate + ? extractCounts(optimizedQueryString, errors, state) + : countRows(optimizedQueryString, errors, state); + + Function unoptimizedQuery = state -> extractCounts(unoptimizedQueryString, errors, state); + reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery); + String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, @@ -81,6 +115,11 @@ public String getLastQueryString() { return lastQueryString; } + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 8f99da8e5..d84f399a4 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -6,6 +6,7 @@ import sqlancer.IgnoreMeException; import sqlancer.LikeImplementationHelper; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; import sqlancer.common.visitor.BinaryOperation; import sqlancer.common.visitor.UnaryOperation; import sqlancer.sqlite3.SQLite3CollateHelper; @@ -18,7 +19,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public abstract class SQLite3Expression { +public abstract class SQLite3Expression implements Expression { public static class SQLite3TableReference extends SQLite3Expression { @@ -128,7 +129,8 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } - public static class Join extends SQLite3Expression { + public static class Join extends SQLite3Expression + implements sqlancer.common.ast.newast.Join { public enum JoinType { INNER, CROSS, OUTER, NATURAL, RIGHT, FULL; @@ -163,6 +165,7 @@ public SQLite3Table getTable() { return table; } + @Override public SQLite3Expression getOnClause() { return onClause; } @@ -176,6 +179,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { return null; } + @Override public void setOnClause(SQLite3Expression onClause) { this.onClause = onClause; } @@ -183,7 +187,6 @@ public void setOnClause(SQLite3Expression onClause) { public void setType(JoinType type) { this.type = type; } - } public static class Subquery extends SQLite3Expression { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 36beb5fd8..57eb91b0b 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -4,9 +4,15 @@ import java.util.Collections; import java.util.List; +import sqlancer.common.ast.newast.Select; +import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column.SQLite3CollateSequence; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3Select extends SQLite3Expression { +public class SQLite3Select extends SQLite3Expression + implements Select { private SelectType fromOptions = SelectType.ALL; private List fromList = Collections.emptyList(); @@ -54,66 +60,82 @@ public void setFromOptions(SelectType fromOptions) { this.fromOptions = fromOptions; } + @Override public List getFromList() { return fromList; } + @Override public void setFromList(List fromList) { this.fromList = fromList; } + @Override public SQLite3Expression getWhereClause() { return whereClause; } + @Override public void setWhereClause(SQLite3Expression whereClause) { this.whereClause = whereClause; } + @Override public void setGroupByClause(List groupByClause) { this.groupByClause = groupByClause; } + @Override public List getGroupByClause() { return groupByClause; } + @Override public void setLimitClause(SQLite3Expression limitClause) { this.limitClause = limitClause; } + @Override public SQLite3Expression getLimitClause() { return limitClause; } + @Override public List getOrderByClauses() { return orderByClause; } + @Override public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } + @Override public void setOffsetClause(SQLite3Expression offsetClause) { this.offsetClause = offsetClause; } + @Override public SQLite3Expression getOffsetClause() { return offsetClause; } + @Override public void setFetchColumns(List fetchColumns) { this.fetchColumns = fetchColumns; } + @Override public List getFetchColumns() { return fetchColumns; } + @Override public void setJoinClauses(List joinStatements) { this.joinStatements = joinStatements; } + @Override public List getJoinClauses() { return joinStatements; } @@ -124,13 +146,19 @@ public SQLite3CollateSequence getExplicitCollateSequence() { return null; } + @Override public void setHavingClause(SQLite3Expression havingClause) { this.havingClause = havingClause; } + @Override public SQLite3Expression getHavingClause() { assert orderByClause != null; return havingClause; } + @Override + public String asString() { + return SQLite3Visitor.asString(this); + } } diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 89e7e5860..7b2b164e6 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -7,7 +7,10 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Select; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Aggregate.SQLite3AggregateFunction; @@ -31,12 +34,14 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; import sqlancer.sqlite3.ast.SQLite3Function.ComputableFunction; import sqlancer.sqlite3.ast.SQLite3RowValueExpression; +import sqlancer.sqlite3.ast.SQLite3Select; import sqlancer.sqlite3.ast.SQLite3UnaryOperation; import sqlancer.sqlite3.ast.SQLite3UnaryOperation.UnaryOperator; import sqlancer.sqlite3.oracle.SQLite3RandomQuerySynthesizer; @@ -45,12 +50,14 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3RowValue; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3ExpressionGenerator implements ExpressionGenerator { +public class SQLite3ExpressionGenerator implements ExpressionGenerator, + NoRECGenerator { private SQLite3RowValue rw; private final SQLite3GlobalState globalState; private boolean tryToGenerateKnownResult; private List columns = Collections.emptyList(); + private List targetTables; private final Randomly r; private boolean deterministicOnly; private boolean allowMatchClause; @@ -63,6 +70,7 @@ public SQLite3ExpressionGenerator(SQLite3ExpressionGenerator other) { this.globalState = other.globalState; this.tryToGenerateKnownResult = other.tryToGenerateKnownResult; this.columns = new ArrayList<>(other.columns); + this.targetTables = other.targetTables; this.r = other.r; this.deterministicOnly = other.deterministicOnly; this.allowMatchClause = other.allowMatchClause; @@ -702,4 +710,71 @@ public SQLite3Expression generateResultKnownExpression() { return expr; } + @Override + public SQLite3ExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(this); + gen.targetTables = targetTables.getTables(); + gen.columns = targetTables.getColumns(); + return gen; + } + + @Override + public SQLite3Expression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public SQLite3Select generateSelect() { + return new SQLite3Select(); + } + + @Override + public List getRandomJoinClauses() { + return getRandomJoinClauses(targetTables); + } + + @Override + public List getTableRefs() { + List tableRefs = new ArrayList<>(); + for (SQLite3Table t : targetTables) { + SQLite3TableReference tableRef; + if (Randomly.getBooleanWithSmallProbability() && !globalState.getSchema().getIndexNames().isEmpty()) { + tableRef = new SQLite3TableReference(globalState.getSchema().getRandomIndexOrBailout(), t); + } else { + tableRef = new SQLite3TableReference(t); + } + tableRefs.add(tableRef); + } + return tableRefs; + } + + @Override + public String generateOptimizedQueryString(Select select, + SQLite3Expression whereCondition, boolean shouldUseAggregate) { + if (Randomly.getBoolean()) { + select.setOrderByClauses(generateOrderBys()); + } + if (shouldUseAggregate) { + select.setFetchColumns(Arrays.asList(new SQLite3Aggregate(Collections.emptyList(), + SQLite3Aggregate.SQLite3AggregateFunction.COUNT_ALL))); + } else { + SQLite3ColumnName aggr = new SQLite3ColumnName(SQLite3Column.createDummy("*"), null); + select.setFetchColumns(Arrays.asList(aggr)); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(Select select, + SQLite3Expression whereCondition) { + SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, + whereCondition); + SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); + select.setFetchColumns(Arrays.asList(asText)); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ")"; + } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 2ec2e392f..27e0229a5 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -1,194 +1,49 @@ package sqlancer.sqlite3.oracle; import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Objects; -import java.util.function.Function; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Visitor; -import sqlancer.sqlite3.ast.SQLite3Aggregate; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.Join; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation.PostfixUnaryOperator; -import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; -public class SQLite3NoRECOracle extends NoRECBase implements TestOracle { +public class SQLite3NoRECOracle implements TestOracle { - private static final int NO_VALID_RESULT = -1; - private final SQLite3Schema s; - private SQLite3ExpressionGenerator gen; - private Reproducer reproducer; - - private static class SQLite3NoRECReproducer implements Reproducer { - private final Function optimizedQuery; - private final Function unoptimizedQuery; - - SQLite3NoRECReproducer(Function optimizedQuery, - Function unoptimizedQuery) { - this.optimizedQuery = optimizedQuery; - this.unoptimizedQuery = unoptimizedQuery; - } - - @Override - public boolean bugStillTriggers(SQLite3GlobalState globalState) { - return !Objects.equals(optimizedQuery.apply(globalState), unoptimizedQuery.apply(globalState)); - } - } + NoRECOracle oracle; public SQLite3NoRECOracle(SQLite3GlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - SQLite3Errors.addExpectedExpressionErrors(errors); - SQLite3Errors.addMatchQueryErrors(errors); - SQLite3Errors.addQueryErrors(errors); - errors.add("misuse of aggregate"); - errors.add("misuse of window function"); - errors.add("second argument to nth_value must be a positive integer"); - errors.add("no such table"); - errors.add("no query solution"); - errors.add("unable to use function MATCH in the requested context"); + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors() + .with(SQLite3Errors.getExpectedExpressionErrors().toArray(new String[0])) + .with(SQLite3Errors.getMatchQueryErrors().toArray(new String[0])) + .with(SQLite3Errors.getQueryErrors().toArray(new String[0])) + .with("misuse of aggregate", "misuse of window function", + "second argument to nth_value must be a positive integer", "no such table", "no query solution", + "unable to use function MATCH in the requested context") + .build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - reproducer = null; - SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - gen = new SQLite3ExpressionGenerator(state).setColumns(columns); - SQLite3Expression randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List joinStatements = gen.getRandomJoinClauses(tables); - List tableRefs = SQLite3Common.getTableRefs(tables, s); - SQLite3Select select = new SQLite3Select(); - select.setFromList(tableRefs); - select.setJoinClauses(joinStatements); - - Function optimizedQuery = getOptimizedQuery(select, randomWhereCondition); - Function unoptimizedQuery = getUnoptimizedQuery(select, randomWhereCondition); - int optimizedCount = optimizedQuery.apply(state); - int unoptimizedCount = unoptimizedQuery.apply(state); - if (optimizedCount == NO_VALID_RESULT || unoptimizedCount == NO_VALID_RESULT) { - throw new IgnoreMeException(); - } - if (optimizedCount != unoptimizedCount) { - reproducer = new SQLite3NoRECReproducer(optimizedQuery, unoptimizedQuery); - state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); - throw new AssertionError(optimizedCount + " " + unoptimizedCount); - } - + oracle.check(); } @Override public Reproducer getLastReproducer() { - return reproducer; + return oracle.getLastReproducer(); } @Override public String getLastQueryString() { - return optimizedQueryString; - } - - private Function getUnoptimizedQuery(SQLite3Select select, - SQLite3Expression randomWhereCondition) throws SQLException { - SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, - randomWhereCondition); - SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); - select.setFetchColumns(Arrays.asList(asText)); - select.setWhereClause(null); - unoptimizedQueryString = "SELECT SUM(count) FROM (" + SQLite3Visitor.asString(select) + ")"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - return state -> extractCounts(q, state); + return oracle.getLastQueryString(); } - - private Function getOptimizedQuery(SQLite3Select select, - SQLite3Expression randomWhereCondition) throws SQLException { - boolean useAggregate = Randomly.getBoolean(); - if (Randomly.getBoolean()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - if (useAggregate) { - select.setFetchColumns(Arrays.asList(new SQLite3Aggregate(Collections.emptyList(), - SQLite3Aggregate.SQLite3AggregateFunction.COUNT_ALL))); - } else { - SQLite3ColumnName aggr = new SQLite3ColumnName(SQLite3Column.createDummy("*"), null); - select.setFetchColumns(Arrays.asList(aggr)); - } - select.setWhereClause(randomWhereCondition); - optimizedQueryString = SQLite3Visitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - return state -> useAggregate ? extractCounts(q, state) : countRows(q, state); - } - - private int countRows(SQLQueryAdapter q, SQLite3GlobalState globalState) { - int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return NO_VALID_RESULT; - } else { - try { - while (rs.next()) { - count++; - } - } catch (SQLException e) { - count = NO_VALID_RESULT; - } - } - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw (IgnoreMeException) e; - } - throw new AssertionError(unoptimizedQueryString, e); - } - return count; - } - - private int extractCounts(SQLQueryAdapter q, SQLite3GlobalState globalState) { - int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return NO_VALID_RESULT; - } else { - try { - while (rs.next()) { - count += rs.getInt(1); - } - } catch (SQLException e) { - count = NO_VALID_RESULT; - } - } - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw (IgnoreMeException) e; - } - throw new AssertionError(unoptimizedQueryString, e); - } - return count; - } - } From 8dbfa2887823b16ebeb4bd57d12403f11fa91fc9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 28 Mar 2024 17:19:16 +0800 Subject: [PATCH 423/856] Add NoREC test for Postgres --- .github/workflows/main.yml | 5 +++- test/sqlancer/dbms/TestPostgresNoREC.java | 23 ++++++++++++++++++ test/sqlancer/dbms/TestPostgresPQS.java | 24 +++++++++++++++++++ ...TestPostgres.java => TestPostgresTLP.java} | 16 ++----------- 4 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 test/sqlancer/dbms/TestPostgresNoREC.java create mode 100644 test/sqlancer/dbms/TestPostgresPQS.java rename test/sqlancer/dbms/{TestPostgres.java => TestPostgresTLP.java} (53%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f618ab1ca..8a848716a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -368,7 +368,10 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgres test + run: | + POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresPQS test + POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresTLP test + POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresNoREC test sqlite: name: DBMS Tests (SQLite) diff --git a/test/sqlancer/dbms/TestPostgresNoREC.java b/test/sqlancer/dbms/TestPostgresNoREC.java new file mode 100644 index 000000000..53100e3c3 --- /dev/null +++ b/test/sqlancer/dbms/TestPostgresNoREC.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPostgresNoREC { + + String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); + boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); + + @Test + public void testNoREC() { + assumeTrue(postgresIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", + "false", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestPostgresPQS.java b/test/sqlancer/dbms/TestPostgresPQS.java new file mode 100644 index 000000000..ebf4be1aa --- /dev/null +++ b/test/sqlancer/dbms/TestPostgresPQS.java @@ -0,0 +1,24 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPostgresPQS { + + String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); + boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); + + @Test + public void testPQS() { + assumeTrue(postgresIsAvailable); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", + "ALPHANUMERIC_SPECIALCHAR", "postgres", "--test-collations", "false", "--oracle", "pqs" })); + } + +} diff --git a/test/sqlancer/dbms/TestPostgres.java b/test/sqlancer/dbms/TestPostgresTLP.java similarity index 53% rename from test/sqlancer/dbms/TestPostgres.java rename to test/sqlancer/dbms/TestPostgresTLP.java index 5eeb1f858..e704724b8 100644 --- a/test/sqlancer/dbms/TestPostgres.java +++ b/test/sqlancer/dbms/TestPostgresTLP.java @@ -7,29 +7,17 @@ import sqlancer.Main; -public class TestPostgres { +public class TestPostgresTLP { String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); @Test - public void testPostgres() { + public void testTLP() { assumeTrue(postgresIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", "false" })); } - - @Test - public void testPQS() { - assumeTrue(postgresIsAvailable); - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", - "ALPHANUMERIC_SPECIALCHAR", "--database-prefix", - "pqsdb" /* Workaround for connections not being closed */, "postgres", "--test-collations", - "false", "--oracle", "pqs" })); - } - } From 73e72eceecfda951ca3807d1603f612792b15075 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 29 Mar 2024 09:55:19 +0800 Subject: [PATCH 424/856] Refactor environment check in tests --- test/sqlancer/dbms/TestConfig.java | 7 +++++++ test/sqlancer/dbms/TestPostgresNoREC.java | 5 +---- test/sqlancer/dbms/TestPostgresPQS.java | 5 +---- test/sqlancer/dbms/TestPostgresTLP.java | 5 +---- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 9fac06920..8cea624c7 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -3,4 +3,11 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + + public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; + + public static boolean isEnvironmentTrue(String key) { + String value = System.getenv(key); + return value != null && value.equalsIgnoreCase("true"); + } } diff --git a/test/sqlancer/dbms/TestPostgresNoREC.java b/test/sqlancer/dbms/TestPostgresNoREC.java index 53100e3c3..8b9f00d48 100644 --- a/test/sqlancer/dbms/TestPostgresNoREC.java +++ b/test/sqlancer/dbms/TestPostgresNoREC.java @@ -9,12 +9,9 @@ public class TestPostgresNoREC { - String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); - boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); - @Test public void testNoREC() { - assumeTrue(postgresIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", diff --git a/test/sqlancer/dbms/TestPostgresPQS.java b/test/sqlancer/dbms/TestPostgresPQS.java index ebf4be1aa..f6a37f533 100644 --- a/test/sqlancer/dbms/TestPostgresPQS.java +++ b/test/sqlancer/dbms/TestPostgresPQS.java @@ -9,12 +9,9 @@ public class TestPostgresPQS { - String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); - boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); - @Test public void testPQS() { - assumeTrue(postgresIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", diff --git a/test/sqlancer/dbms/TestPostgresTLP.java b/test/sqlancer/dbms/TestPostgresTLP.java index e704724b8..5bd722991 100644 --- a/test/sqlancer/dbms/TestPostgresTLP.java +++ b/test/sqlancer/dbms/TestPostgresTLP.java @@ -9,12 +9,9 @@ public class TestPostgresTLP { - String postgresAvailable = System.getenv("POSTGRES_AVAILABLE"); - boolean postgresIsAvailable = postgresAvailable != null && postgresAvailable.equalsIgnoreCase("true"); - @Test public void testTLP() { - assumeTrue(postgresIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", From b6fb5ad5ac86f1c7b7ff87474d401545034a1ac3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 28 Mar 2024 15:31:48 +0800 Subject: [PATCH 425/856] Implement expression interface for Postgres expressions --- src/sqlancer/common/ast/SelectBase.java | 11 +++++++++++ src/sqlancer/mariadb/ast/MariaDBSelectStatement.java | 1 + src/sqlancer/oceanbase/ast/OceanBaseSelect.java | 2 ++ src/sqlancer/postgres/ast/PostgresExpression.java | 4 +++- src/sqlancer/postgres/ast/PostgresJoin.java | 6 +++++- src/sqlancer/postgres/ast/PostgresSelect.java | 12 +++++++++++- 6 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/common/ast/SelectBase.java b/src/sqlancer/common/ast/SelectBase.java index ebc3ed5c3..e79a1a87d 100644 --- a/src/sqlancer/common/ast/SelectBase.java +++ b/src/sqlancer/common/ast/SelectBase.java @@ -36,6 +36,10 @@ public void setFromList(List fromList) { this.fromList = fromList; } + public void setFromTables(List tables) { + setFromList(tables); + } + public List getFromList() { if (fromList == null) { throw new IllegalStateException(); @@ -115,4 +119,11 @@ public void setJoinList(List joinList) { this.joinList = joinList; } + public List getGroupByClause() { + return getGroupByExpressions(); + } + + public void setGroupByClause(List groupByExpressions) { + setGroupByExpressions(groupByExpressions); + } } diff --git a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java index a690276d1..813b92337 100644 --- a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java +++ b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java @@ -16,6 +16,7 @@ public enum MariaDBSelectType { private MariaDBSelectType selectType = MariaDBSelectType.ALL; private MariaDBExpression whereCondition; + @Override public void setGroupByClause(List groupBys) { this.groupBys = groupBys; } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java index a87ff5c27..a673492c2 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java @@ -29,10 +29,12 @@ public void setFromOptions(SelectType fromOptions) { this.fromOptions = fromOptions; } + @Override public void setGroupByClause(List groupBys) { this.groupBys = groupBys; } + @Override public List getGroupByClause() { return this.groupBys; } diff --git a/src/sqlancer/postgres/ast/PostgresExpression.java b/src/sqlancer/postgres/ast/PostgresExpression.java index 96ddfa433..433e2207c 100644 --- a/src/sqlancer/postgres/ast/PostgresExpression.java +++ b/src/sqlancer/postgres/ast/PostgresExpression.java @@ -1,8 +1,10 @@ package sqlancer.postgres.ast; +import sqlancer.common.ast.newast.Expression; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; -public interface PostgresExpression { +public interface PostgresExpression extends Expression { default PostgresDataType getExpressionType() { return null; diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index d1a076bbe..7bfef3f6f 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -5,12 +5,14 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class PostgresJoin implements PostgresExpression { +public class PostgresJoin implements PostgresExpression, Join { public enum PostgresJoinType { INNER, LEFT, RIGHT, FULL, CROSS; @@ -77,6 +79,7 @@ public static List getJoins(List tableLi return joinExpressions; } + @Override public void setOnClause(PostgresExpression clause) { this.onClause = clause; } @@ -97,6 +100,7 @@ public PostgresExpression getRightTable() { return rightTable; } + @Override public PostgresExpression getOnClause() { return onClause; } diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index 70a42650f..c864a56e8 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -5,10 +5,14 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresVisitor; -public class PostgresSelect extends SelectBase implements PostgresExpression { +public class PostgresSelect extends SelectBase + implements PostgresExpression, Select { private SelectType selectOption = SelectType.ALL; private List joinClauses = Collections.emptyList(); @@ -111,11 +115,13 @@ public PostgresDataType getExpressionType() { return null; } + @Override public void setJoinClauses(List joinStatements) { this.joinClauses = joinStatements; } + @Override public List getJoinClauses() { return joinClauses; } @@ -132,4 +138,8 @@ public ForClause getForClause() { return forClause; } + @Override + public String asString() { + return PostgresVisitor.asString(this); + } } From d9def9379ca8951cb69bf578fa17dda4390fdc73 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 28 Mar 2024 17:08:29 +0800 Subject: [PATCH 426/856] Specify select type for NoREC generator --- src/sqlancer/common/gen/NoRECGenerator.java | 10 +++++----- src/sqlancer/common/oracle/NoRECOracle.java | 8 ++++---- .../sqlite3/gen/SQLite3ExpressionGenerator.java | 10 ++++------ src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java | 3 ++- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/sqlancer/common/gen/NoRECGenerator.java b/src/sqlancer/common/gen/NoRECGenerator.java index f57f7c20f..185e8286b 100644 --- a/src/sqlancer/common/gen/NoRECGenerator.java +++ b/src/sqlancer/common/gen/NoRECGenerator.java @@ -9,13 +9,13 @@ import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; -public interface NoRECGenerator, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { +public interface NoRECGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { - NoRECGenerator setTablesAndColumns(AbstractTables tables); + NoRECGenerator setTablesAndColumns(AbstractTables tables); E generateBooleanExpression(); - Select generateSelect(); + S generateSelect(); List getRandomJoinClauses(); @@ -33,7 +33,7 @@ public interface NoRECGenerator, E extends Expression * * @return a query string to be executed */ - String generateOptimizedQueryString(Select select, E whereCondition, boolean shouldUseAggregate); + String generateOptimizedQueryString(S select, E whereCondition, boolean shouldUseAggregate); /** * Generates a query string that is unlikely to be optimized by the DBMS. @@ -45,5 +45,5 @@ public interface NoRECGenerator, E extends Expression * * @return a query string to be executed */ - String generateUnoptimizedQueryString(Select select, E whereCondition); + String generateUnoptimizedQueryString(S select, E whereCondition); } diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index aac047b2c..acc7cffac 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -20,12 +20,12 @@ import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; -public class NoRECOracle, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> +public class NoRECOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> implements TestOracle { private final G state; - private NoRECGenerator gen; + private NoRECGenerator gen; private final ExpectedErrors errors; private Reproducer reproducer; @@ -46,7 +46,7 @@ public boolean bugStillTriggers(G globalState) { } } - public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); } @@ -63,7 +63,7 @@ public void check() throws SQLException { AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); gen = gen.setTablesAndColumns(targetTables); - Select select = gen.generateSelect(); + Z select = gen.generateSelect(); select.setJoinClauses(gen.getRandomJoinClauses()); select.setFromList(gen.getTableRefs()); diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 7b2b164e6..0ac0cd272 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -7,7 +7,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Select; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.schema.AbstractTables; @@ -51,7 +50,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; public class SQLite3ExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator { private SQLite3RowValue rw; private final SQLite3GlobalState globalState; @@ -749,8 +748,8 @@ public List getTableRefs() { } @Override - public String generateOptimizedQueryString(Select select, - SQLite3Expression whereCondition, boolean shouldUseAggregate) { + public String generateOptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition, + boolean shouldUseAggregate) { if (Randomly.getBoolean()) { select.setOrderByClauses(generateOrderBys()); } @@ -767,8 +766,7 @@ public String generateOptimizedQueryString(Select select, - SQLite3Expression whereCondition) { + public String generateUnoptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition) { SQLite3PostfixUnaryOperation isTrue = new SQLite3PostfixUnaryOperation(PostfixUnaryOperator.IS_TRUE, whereCondition); SQLite3PostfixText asText = new SQLite3PostfixText(isTrue, " as count", null); diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 27e0229a5..7393ec8d5 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -10,6 +10,7 @@ import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.ast.SQLite3Select; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; @@ -17,7 +18,7 @@ public class SQLite3NoRECOracle implements TestOracle { - NoRECOracle oracle; + NoRECOracle oracle; public SQLite3NoRECOracle(SQLite3GlobalState globalState) { SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); From 85c22fe3179ef68e9abd898bc8eac97de6558fbb Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 28 Mar 2024 17:09:22 +0800 Subject: [PATCH 427/856] Implement NoREC generator for Postgres --- .../gen/PostgresExpressionGenerator.java | 122 +++++++++++++++++- 1 file changed, 121 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index f979e97da..5b5851671 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -11,12 +11,16 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresRowValue; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.PostgresSchema.PostgresTables; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresAggregate.PostgresAggregateFunction; import sqlancer.postgres.ast.PostgresBetweenOperation; @@ -40,6 +44,8 @@ import sqlancer.postgres.ast.PostgresFunction.PostgresFunctionWithResult; import sqlancer.postgres.ast.PostgresFunctionWithUnknownResult; import sqlancer.postgres.ast.PostgresInOperation; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresOrderByTerm.PostgresOrder; @@ -47,11 +53,18 @@ import sqlancer.postgres.ast.PostgresPOSIXRegularExpression.POSIXRegex; import sqlancer.postgres.ast.PostgresPostfixOperation; import sqlancer.postgres.ast.PostgresPostfixOperation.PostfixOperator; +import sqlancer.postgres.ast.PostgresPostfixText; import sqlancer.postgres.ast.PostgresPrefixOperation; import sqlancer.postgres.ast.PostgresPrefixOperation.PrefixOperator; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.ast.PostgresSelect.ForClause; +import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; +import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; +import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.ast.PostgresSimilarTo; -public class PostgresExpressionGenerator implements ExpressionGenerator { +public class PostgresExpressionGenerator implements ExpressionGenerator, + NoRECGenerator { private final int maxDepth; @@ -59,6 +72,8 @@ public class PostgresExpressionGenerator implements ExpressionGenerator columns; + private List targetTables; + private PostgresRowValue rw; private boolean expectedResult; @@ -586,6 +601,35 @@ public PostgresExpressionGenerator allowAggregates(boolean value) { return this; } + public static PostgresSubquery createSubquery(PostgresGlobalState globalState, String name, PostgresTables tables) { + List columns = new ArrayList<>(); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + PostgresSelect select = new PostgresSelect(); + select.setFromList(tables.getTables().stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause( + PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(ForClause.getRandom()); + } + return new PostgresSubquery(select, name); + } + @Override public PostgresExpression generatePredicate() { return generateExpression(PostgresDataType.BOOLEAN); @@ -601,4 +645,80 @@ public PostgresExpression isNull(PostgresExpression expr) { return new PostgresPostfixOperation(expr, PostfixOperator.IS_NULL); } + @Override + public PostgresExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + this.targetTables = targetTables.getTables(); + this.columns = targetTables.getColumns(); + return this; + } + + @Override + public PostgresExpression generateBooleanExpression() { + return generateExpression(PostgresDataType.BOOLEAN); + } + + @Override + public PostgresSelect generateSelect() { + return new PostgresSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + for (int i = 1; i < targetTables.size(); i++) { + PostgresExpression joinClause = generateExpression(PostgresDataType.BOOLEAN); + PostgresTable table = Randomly.fromList(targetTables); + targetTables.remove(table); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + PostgresTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PostgresSubquery subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); + PostgresExpression joinClause = generateExpression(PostgresDataType.BOOLEAN); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public List getTableRefs() { + return targetTables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(PostgresSelect select, PostgresExpression whereCondition, + boolean shouldUseAggregate) { + PostgresColumnValue allColumns = new PostgresColumnValue(PostgresColumn.createDummy("*"), null); + if (shouldUseAggregate) { + select.setFetchColumns( + Arrays.asList(new PostgresAggregate(List.of(allColumns), PostgresAggregateFunction.COUNT))); + } else { + select.setFetchColumns(Arrays.asList(allColumns)); + } + select.setWhereClause(whereCondition); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBy()); + } + select.setSelectType(SelectType.ALL); + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(PostgresSelect select, PostgresExpression whereCondition) { + PostgresCastOperation isTrue = new PostgresCastOperation(whereCondition, + PostgresCompoundDataType.create(PostgresDataType.INT)); + PostgresPostfixText asText = new PostgresPostfixText(isTrue, " as count", null, PostgresDataType.INT); + select.setFetchColumns(Arrays.asList(asText)); + select.setWhereClause(null); + select.setOrderByClauses(List.of()); + select.setSelectType(SelectType.ALL); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } From 0931c3feb96c054d1138989394056e2361449dbf Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 29 Mar 2024 10:38:58 +0800 Subject: [PATCH 428/856] Use generic NoREC oracle for Postgres --- .../postgres/oracle/PostgresNoRECOracle.java | 131 +++--------------- 1 file changed, 22 insertions(+), 109 deletions(-) diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index 2ad7cc2b7..d1553c35f 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -1,77 +1,57 @@ package sqlancer.postgres.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; +import java.util.regex.Pattern; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.PostgresCompoundDataType; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresSchema.PostgresTables; -import sqlancer.postgres.PostgresVisitor; -import sqlancer.postgres.ast.PostgresCastOperation; -import sqlancer.postgres.ast.PostgresColumnValue; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; -import sqlancer.postgres.ast.PostgresPostfixText; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; -import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; import sqlancer.postgres.oracle.tlp.PostgresTLPBase; -public class PostgresNoRECOracle extends NoRECBase implements TestOracle { +public class PostgresNoRECOracle implements TestOracle { - private final PostgresSchema s; + private final NoRECOracle oracle; public PostgresNoRECOracle(PostgresGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - PostgresCommon.addCommonExpressionErrors(errors); - PostgresCommon.addCommonFetchErrors(errors); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors() + .with(PostgresCommon.getCommonExpressionErrors().toArray(new String[0])) + .with(PostgresCommon.getCommonFetchErrors().toArray(new String[0])) + .with(PostgresCommon.getCommonExpressionRegexErrors().toArray(new Pattern[0])).build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - PostgresTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - PostgresExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); + oracle.check(); + } - List joinStatements = getJoinStatements(state, columns, tables); - List fromTables = tables.stream().map(t -> new PostgresFromTable(t, Randomly.getBoolean())) - .collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } public static List getJoinStatements(PostgresGlobalState globalState, List columns, @@ -98,71 +78,4 @@ public static List getJoinStatements(PostgresGlobalState globalSta } return joinStatements; } - - private PostgresExpression getRandomWhereCondition(List columns) { - return new PostgresExpressionGenerator(state).setColumns(columns).generateExpression(PostgresDataType.BOOLEAN); - } - - private int getUnoptimizedQueryCount(List fromTables, PostgresExpression randomWhereCondition, - List joinStatements) throws SQLException { - PostgresSelect select = new PostgresSelect(); - PostgresCastOperation isTrue = new PostgresCastOperation(randomWhereCondition, - PostgresCompoundDataType.create(PostgresDataType.INT)); - PostgresPostfixText asText = new PostgresPostfixText(isTrue, " as count", null, PostgresDataType.INT); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(fromTables); - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + PostgresVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(List randomTables, List columns, - PostgresExpression randomWhereCondition, List joinStatements) throws SQLException { - PostgresSelect select = new PostgresSelect(); - PostgresColumnValue allColumns = new PostgresColumnValue(Randomly.fromList(columns), null); - select.setFetchColumns(Arrays.asList(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new PostgresExpressionGenerator(state).setColumns(columns).generateOrderBy()); - } - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = PostgresVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; - } - } From 1e01d0a5dae53cdacecff107bbe4a82a8eaf27c8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 29 Mar 2024 10:39:24 +0800 Subject: [PATCH 429/856] Use generic NoREC oracle for Citus --- .../citus/oracle/CitusNoRECOracle.java | 43 +++++++++++++++++-- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/citus/oracle/CitusNoRECOracle.java b/src/sqlancer/citus/oracle/CitusNoRECOracle.java index 88ec3391a..a59207ec4 100644 --- a/src/sqlancer/citus/oracle/CitusNoRECOracle.java +++ b/src/sqlancer/citus/oracle/CitusNoRECOracle.java @@ -1,14 +1,49 @@ package sqlancer.citus.oracle; +import java.sql.SQLException; +import java.util.regex.Pattern; + +import sqlancer.Reproducer; import sqlancer.citus.gen.CitusCommon; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.oracle.PostgresNoRECOracle; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class CitusNoRECOracle extends PostgresNoRECOracle { +public class CitusNoRECOracle implements TestOracle { + private final NoRECOracle oracle; public CitusNoRECOracle(PostgresGlobalState globalState) { - super(globalState); - CitusCommon.addCitusErrors(errors); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors() + .with(PostgresCommon.getCommonExpressionErrors().toArray(new String[0])) + .with(PostgresCommon.getCommonFetchErrors().toArray(new String[0])) + .with(PostgresCommon.getCommonExpressionRegexErrors().toArray(new Pattern[0])) + .with(CitusCommon.getCitusErrors().toArray(new String[0])).build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); + } + + @Override + public void check() throws SQLException { + oracle.check(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From b7ac21ab075a9b4111c02af62765630735fd1ec9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sun, 31 Mar 2024 10:18:29 +0800 Subject: [PATCH 430/856] Rename `with` method in ExpectedErrorsBuilder for regex & regex string --- .../citus/oracle/CitusNoRECOracle.java | 7 ++--- src/sqlancer/common/query/ExpectedErrors.java | 16 +++++++++-- .../postgres/oracle/PostgresNoRECOracle.java | 8 ++---- .../sqlite3/oracle/SQLite3NoRECOracle.java | 6 ++-- test/sqlancer/TestExpectedErrors.java | 28 +++++++++++++++++-- 5 files changed, 47 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/citus/oracle/CitusNoRECOracle.java b/src/sqlancer/citus/oracle/CitusNoRECOracle.java index a59207ec4..ffc7c13fb 100644 --- a/src/sqlancer/citus/oracle/CitusNoRECOracle.java +++ b/src/sqlancer/citus/oracle/CitusNoRECOracle.java @@ -1,7 +1,6 @@ package sqlancer.citus.oracle; import java.sql.SQLException; -import java.util.regex.Pattern; import sqlancer.Reproducer; import sqlancer.citus.gen.CitusCommon; @@ -23,10 +22,8 @@ public class CitusNoRECOracle implements TestOracle { public CitusNoRECOracle(PostgresGlobalState globalState) { PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors() - .with(PostgresCommon.getCommonExpressionErrors().toArray(new String[0])) - .with(PostgresCommon.getCommonFetchErrors().toArray(new String[0])) - .with(PostgresCommon.getCommonExpressionRegexErrors().toArray(new Pattern[0])) + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) .with(CitusCommon.getCitusErrors().toArray(new String[0])).build(); this.oracle = new NoRECOracle<>(globalState, gen, errors); } diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index 348da179f..f71d185e1 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -110,18 +110,30 @@ public ExpectedErrorsBuilder with(String... list) { return this; } - public ExpectedErrorsBuilder with(Pattern... list) { + public ExpectedErrorsBuilder with(Collection list) { + return with(list.toArray(new String[0])); + } + + public ExpectedErrorsBuilder withRegex(Pattern... list) { regexes.addAll(Arrays.asList(list)); return this; } - public ExpectedErrorsBuilder withRegex(String... list) { + public ExpectedErrorsBuilder withRegex(Collection list) { + return withRegex(list.toArray(new Pattern[0])); + } + + public ExpectedErrorsBuilder withRegexString(String... list) { for (String error : list) { regexes.add(Pattern.compile(error)); } return this; } + public ExpectedErrorsBuilder withRegexString(Collection list) { + return withRegexString(list.toArray(new String[0])); + } + public ExpectedErrors build() { return new ExpectedErrors(errors, regexes); } diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java index d1553c35f..9243b4870 100644 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java @@ -3,7 +3,6 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; -import java.util.regex.Pattern; import sqlancer.Randomly; import sqlancer.Reproducer; @@ -32,10 +31,9 @@ public class PostgresNoRECOracle implements TestOracle { public PostgresNoRECOracle(PostgresGlobalState globalState) { PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors() - .with(PostgresCommon.getCommonExpressionErrors().toArray(new String[0])) - .with(PostgresCommon.getCommonFetchErrors().toArray(new String[0])) - .with(PostgresCommon.getCommonExpressionRegexErrors().toArray(new Pattern[0])).build(); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .build(); this.oracle = new NoRECOracle<>(globalState, gen, errors); } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java index 7393ec8d5..8f6e2d276 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java @@ -22,10 +22,8 @@ public class SQLite3NoRECOracle implements TestOracle { public SQLite3NoRECOracle(SQLite3GlobalState globalState) { SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors() - .with(SQLite3Errors.getExpectedExpressionErrors().toArray(new String[0])) - .with(SQLite3Errors.getMatchQueryErrors().toArray(new String[0])) - .with(SQLite3Errors.getQueryErrors().toArray(new String[0])) + ExpectedErrors errors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .with(SQLite3Errors.getMatchQueryErrors()).with(SQLite3Errors.getQueryErrors()) .with("misuse of aggregate", "misuse of window function", "second argument to nth_value must be a positive integer", "no such table", "no query solution", "unable to use function MATCH in the requested context") diff --git a/test/sqlancer/TestExpectedErrors.java b/test/sqlancer/TestExpectedErrors.java index 2cb938824..2a3b1a67e 100644 --- a/test/sqlancer/TestExpectedErrors.java +++ b/test/sqlancer/TestExpectedErrors.java @@ -118,14 +118,38 @@ public void testBuilder() { assertTrue(errors.errorIsExpected("aa")); assertFalse(errors.errorIsExpected("d")); - errors = ExpectedErrors.newErrors().withRegex("a\\d", "b\\D").with("c").build(); + errors = ExpectedErrors.newErrors().with(List.of("a", "b", "c")).build(); + + assertTrue(errors.errorIsExpected("a")); + assertTrue(errors.errorIsExpected("b")); + assertTrue(errors.errorIsExpected("c")); + assertTrue(errors.errorIsExpected("aa")); + assertFalse(errors.errorIsExpected("d")); + + errors = ExpectedErrors.newErrors().withRegexString("a\\d", "b\\D").with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + errors = ExpectedErrors.newErrors().withRegexString(List.of("a\\d", "b\\D")).with("c").build(); + + assertTrue(errors.errorIsExpected("a0")); + assertTrue(errors.errorIsExpected("bb")); + assertTrue(errors.errorIsExpected("c")); + assertFalse(errors.errorIsExpected("aa")); + + errors = ExpectedErrors.newErrors().withRegex(Pattern.compile("a\\d"), Pattern.compile("b\\D")).with("c") + .build(); assertTrue(errors.errorIsExpected("a0")); assertTrue(errors.errorIsExpected("bb")); assertTrue(errors.errorIsExpected("c")); assertFalse(errors.errorIsExpected("aa")); - errors = ExpectedErrors.newErrors().with(Pattern.compile("a\\d"), Pattern.compile("b\\D")).with("c").build(); + errors = ExpectedErrors.newErrors().withRegex(List.of(Pattern.compile("a\\d"), Pattern.compile("b\\D"))) + .with("c").build(); assertTrue(errors.errorIsExpected("a0")); assertTrue(errors.errorIsExpected("bb")); From bd511f09846cc585e8f7f4a620a65e08482e1528 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 1 Apr 2024 14:17:58 +0800 Subject: [PATCH 431/856] Pin Clickhouse version to 23.1.2672 lts --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f0adb6e4..766aac668 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -109,8 +109,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up ClickHouse run: | - docker pull clickhouse/clickhouse-server:head - docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:head + docker pull clickhouse/clickhouse-server:24.3.1.2672 + docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:24.3.1.2672 sleep 5 - name: Run Tests run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test From 27638c3be1543094b97288ddfb9516572d39c0f8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 1 Apr 2024 14:48:39 +0800 Subject: [PATCH 432/856] Update ExpectedErrors to match change in error --- src/sqlancer/clickhouse/ClickHouseErrors.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 23c27c8bf..c4ccd7e44 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -34,9 +34,8 @@ public static List getExpectedExpressionErrors() { "is not under aggregate function and not in GROUP BY", "is not under aggregate function", "is violated at row 1. Expression:", // TODO: check constraint on table creation "is violated, because it is a constant expression returning 0. It is most likely an error in table definition", - "there are only columns", "there are columns", "in block. (NOT_FOUND_COLUMN_IN_BLOCK)", - "Missing columns", "Ambiguous column", - "Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)", + "there are only columns", "there are columns", "(NOT_FOUND_COLUMN_IN_BLOCK)", "Missing columns", + "Ambiguous column", "Must be one unsigned integer type. (ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER)", "Floating point partition key is not supported", "Cannot get JOIN keys from JOIN ON section", "ILLEGAL_DIVISION", "DECIMAL_OVERFLOW", "Cannot convert out of range floating point value to integer type", From d6ab9965e746d69b67d63f7aaa2738f9370f82b8 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Wed, 3 Apr 2024 21:11:46 +0800 Subject: [PATCH 433/856] [MySQL] Add bugs 114533 and 114544 (#938) * Pass exception from ResultSet up as SQLException * [MySQL] Add bugs 114533 and 114534 --- src/sqlancer/common/query/SQLancerResultSet.java | 6 +++++- src/sqlancer/mysql/MySQLBugs.java | 6 ++++++ src/sqlancer/mysql/MySQLErrors.java | 13 +++++++++++++ .../mysql/gen/datadef/MySQLIndexGenerator.java | 7 ++++++- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/query/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java index d6f2a67cb..d1221a7f7 100644 --- a/src/sqlancer/common/query/SQLancerResultSet.java +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -35,7 +35,11 @@ public int getInt(int i) throws SQLException { } public String getString(int i) throws SQLException { - return rs.getString(i); + try { + return rs.getString(i); + } catch (NumberFormatException e) { + throw new SQLException(e); + } } public boolean isClosed() throws SQLException { diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 00a7a0ddf..8cb8a3391 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -31,6 +31,12 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=112264 public static boolean bug112264 = true; + // https://bugs.mysql.com/bug.php?id=114533 + public static boolean bug114533 = true; + + // https://bugs.mysql.com/bug.php?id=114534 + public static boolean bug114534 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 3f681acb5..9dcc0a5c3 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; import sqlancer.common.query.ExpectedErrors; @@ -23,8 +24,20 @@ public static List getExpressionErrors() { return errors; } + public static List getExpressionRegexErrors() { + ArrayList errors = new ArrayList<>(); + + if (MySQLBugs.bug114533) { + errors.add(Pattern.compile("For input string: \"0+-0\"")); // match: For input string: + // "00000000000000000000-0" + } + + return errors; + } + public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); + errors.addAllRegexes(getExpressionRegexErrors()); } public static List getInsertUpdateErrors() { diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index 3f418385d..550893db5 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -5,6 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -78,7 +79,11 @@ public SQLQueryAdapter create() { if (Randomly.getBoolean() && c.getType() == MySQLDataType.VARCHAR) { sb.append("("); // TODO for string - sb.append(r.getInteger(1, 5)); + if (MySQLBugs.bug114534) { + sb.append(r.getInteger(2, 5)); + } else { + sb.append(r.getInteger(1, 5)); + } sb.append(")"); } if (Randomly.getBoolean()) { From 3c4d04d78b6d5cf0e4807c91028208345417aef2 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Sun, 7 Apr 2024 22:21:13 +0800 Subject: [PATCH 434/856] [MySQL] Pin version to 8.0.36 (#940) --- .github/workflows/main.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 766aac668..dcbb1af99 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -323,6 +323,14 @@ jobs: mysql: name: DBMS Tests (MySQL) runs-on: ubuntu-latest + services: + mysql: + image: mysql:8.0.36 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - uses: actions/checkout@v3 with: @@ -335,11 +343,8 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up MySQL - run: | - sudo systemctl start mysql.service # MySQL 8.0.32-0ubuntu0.22.04.2 - name: Create SQLancer user - run: mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" + run: mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS From 3095ec5f2a5b2005f885ae2d3b95513446767043 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 11:40:49 +0800 Subject: [PATCH 435/856] Add TLPWhereGenerator interface --- .../common/gen/PartitionGenerator.java | 27 ++++++++++++++++++ .../common/gen/TLPWhereGenerator.java | 28 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 src/sqlancer/common/gen/PartitionGenerator.java create mode 100644 src/sqlancer/common/gen/TLPWhereGenerator.java diff --git a/src/sqlancer/common/gen/PartitionGenerator.java b/src/sqlancer/common/gen/PartitionGenerator.java new file mode 100644 index 000000000..affc62c42 --- /dev/null +++ b/src/sqlancer/common/gen/PartitionGenerator.java @@ -0,0 +1,27 @@ +package sqlancer.common.gen; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.schema.AbstractTableColumn; + +public interface PartitionGenerator, C extends AbstractTableColumn> { + + /** + * Negates a predicate (i.e., uses a NOT operator). + * + * @param predicate + * the boolean predicate. + * + * @return the negated predicate. + */ + E negatePredicate(E predicate); + + /** + * Checks if an expression evaluates to NULL (i.e., implements the IS NULL operator). + * + * @param expr + * the expression + * + * @return an expression that checks whether the expression evaluates to NULL. + */ + E isNull(E expr); +} diff --git a/src/sqlancer/common/gen/TLPWhereGenerator.java b/src/sqlancer/common/gen/TLPWhereGenerator.java new file mode 100644 index 000000000..095a878f9 --- /dev/null +++ b/src/sqlancer/common/gen/TLPWhereGenerator.java @@ -0,0 +1,28 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface TLPWhereGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> + extends PartitionGenerator { + + TLPWhereGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + List generateOrderBys(); +} From 307713ac59675bae6286e86eda8290db63e1ddb8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 11:41:04 +0800 Subject: [PATCH 436/856] Add generic TLPWhere oracle --- .../common/oracle/TLPWhereOracle.java | 84 +++++++++++++++++++ .../common/oracle/TestOracleUtils.java | 33 ++++++++ 2 files changed, 117 insertions(+) create mode 100644 src/sqlancer/common/oracle/TLPWhereOracle.java diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java new file mode 100644 index 000000000..5f7b3feb6 --- /dev/null +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -0,0 +1,84 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class TLPWhereOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + + private TLPWhereGenerator gen; + private final ExpectedErrors errors; + + private String generatedQueryString; + + public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + S s = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(s); + gen = gen.setTablesAndColumns(targetTables); + + Select select = gen.generateSelect(); + + boolean shouldCreateDummy = true; + select.setFetchColumns(gen.generateFetchColumns(shouldCreateDummy)); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + select.setWhereClause(null); + + String originalQueryString = select.asString(); + generatedQueryString = originalQueryString; + List firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + boolean orderBy = Randomly.getBooleanWithSmallProbability(); + if (orderBy) { + select.setOrderByClauses(gen.generateOrderBys()); + } + + TestOracleUtils.PredicateVariants predicates = TestOracleUtils.initializeTernaryPredicateVariants(gen, + gen.generateBooleanExpression()); + select.setWhereClause(predicates.predicate); + String firstQueryString = select.asString(); + select.setWhereClause(predicates.negatedPredicate); + String secondQueryString = select.asString(); + select.setWhereClause(predicates.isNullPredicate); + String thirdQueryString = select.asString(); + + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + + ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, + state); + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java index b22cc89f5..bab2e26c9 100644 --- a/src/sqlancer/common/oracle/TestOracleUtils.java +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -2,6 +2,8 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.gen.PartitionGenerator; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; @@ -12,6 +14,18 @@ public final class TestOracleUtils { private TestOracleUtils() { } + public static final class PredicateVariants, C extends AbstractTableColumn> { + public E predicate; + public E negatedPredicate; + public E isNullPredicate; + + PredicateVariants(E predicate, E negatedPredicate, E isNullPredicate) { + this.predicate = predicate; + this.negatedPredicate = negatedPredicate; + this.isNullPredicate = isNullPredicate; + } + } + public static , C extends AbstractTableColumn> AbstractTables getRandomTableNonEmptyTables( AbstractSchema schema) { if (schema.getDatabaseTables().isEmpty()) { @@ -19,4 +33,23 @@ private TestOracleUtils() { } return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables())); } + + public static , T extends AbstractTable, C extends AbstractTableColumn> PredicateVariants initializeTernaryPredicateVariants( + PartitionGenerator gen, E predicate) { + if (gen == null) { + throw new IllegalStateException(); + } + if (predicate == null) { + throw new IllegalStateException(); + } + E negatedPredicate = gen.negatePredicate(predicate); + if (negatedPredicate == null) { + throw new IllegalStateException(); + } + E isNullPredicate = gen.isNull(predicate); + if (isNullPredicate == null) { + throw new IllegalStateException(); + } + return new PredicateVariants<>(predicate, negatedPredicate, isNullPredicate); + } } From 6e4204830d61292333e3ac58621f3dfbc294fde9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 11:59:25 +0800 Subject: [PATCH 437/856] Implement TLPWhereGenerator interfaces for SQLite3 --- .../sqlite3/gen/SQLite3ExpressionGenerator.java | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java index 0ac0cd272..e4f0741f9 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.sqlite3.SQLite3GlobalState; import sqlancer.sqlite3.ast.SQLite3Aggregate; @@ -50,7 +51,8 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; public class SQLite3ExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private SQLite3RowValue rw; private final SQLite3GlobalState globalState; @@ -133,6 +135,7 @@ public static SQLite3Expression getRandomLiteralValue(SQLite3GlobalState globalS return new SQLite3ExpressionGenerator(globalState).getRandomLiteralValueInternal(globalState.getRandomly()); } + @Override public List generateOrderBys() { List expressions = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { @@ -747,6 +750,18 @@ public List getTableRefs() { return tableRefs; } + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + List columns = new ArrayList<>(); + if (shouldCreateDummy && Randomly.getBoolean()) { + columns.add(new SQLite3ColumnName(SQLite3Column.createDummy("*"), null)); + } else { + columns = Randomly.nonEmptySubset(this.columns).stream().map(c -> new SQLite3ColumnName(c, null)) + .collect(Collectors.toList()); + } + return columns; + } + @Override public String generateOptimizedQueryString(SQLite3Select select, SQLite3Expression whereCondition, boolean shouldUseAggregate) { From 17f7d3ee09f267267c08cbdd1120482afe4d7715 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 12:00:14 +0800 Subject: [PATCH 438/856] Use generic TLPWhere oracle for SQLite3 --- .../oracle/tlp/SQLite3TLPWhereOracle.java | 49 +++++++------------ 1 file changed, 18 insertions(+), 31 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java index b9580ddd8..8d850fdcb 100644 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java +++ b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java @@ -1,50 +1,37 @@ package sqlancer.sqlite3.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Expression; +import sqlancer.sqlite3.ast.SQLite3Select; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.schema.SQLite3Schema; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3TLPWhereOracle extends SQLite3TLPBase { +public class SQLite3TLPWhereOracle implements TestOracle { - private String generatedQueryString; + private final TLPWhereOracle oracle; public SQLite3TLPWhereOracle(SQLite3GlobalState state) { - super(state); + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .build(); + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = SQLite3Visitor.asString(select); - generatedQueryString = originalQueryString; - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithSmallProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = SQLite3Visitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = SQLite3Visitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = SQLite3Visitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } @Override public String getLastQueryString() { - return generatedQueryString; + return oracle.getLastQueryString(); } - } From fe7f7939343bbbc019560e3c7f80ffdc855da9d1 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 14:58:42 +0800 Subject: [PATCH 439/856] [MariaDB] Pin version to 11.3.2 --- .github/workflows/main.yml | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dcbb1af99..a2d21b67d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -241,6 +241,14 @@ jobs: mariadb: name: DBMS Tests (MariaDB) runs-on: ubuntu-latest + services: + mysql: + image: mariadb:11.3.2 + env: + MYSQL_ROOT_PASSWORD: root + ports: + - 3306:3306 + options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - uses: actions/checkout@v3 @@ -254,15 +262,8 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Install MariaDB - run: | - sudo apt-key adv --recv-keys --keyserver hkp://keyserver.ubuntu.com:80 0xF1656F24C74CD1D8 - sudo add-apt-repository 'deb [arch=amd64,arm64,ppc64el] http://sfo1.mirrors.digitalocean.com/mariadb/repo/10.3/ubuntu bionic main' - sudo apt update - sudo apt install mariadb-server - sudo systemctl start mariadb - name: Create SQLancer User - run: sudo mysql -uroot -proot -e "CREATE USER 'sqlancer'@'localhost' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'localhost';" + run: sudo mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test From a2f2bb08fbf06c43c0d0663e8932e4a4879bc594 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 16:24:47 +0800 Subject: [PATCH 440/856] [MariaDB] Remove `engine_condition_pushdown=off` optimizer switch --- .../mariadb/gen/MariaDBSetGenerator.java | 23 +++++++++---------- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 11 ++++----- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java index fc1f1d5a6..860f60c4f 100644 --- a/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBSetGenerator.java @@ -122,15 +122,14 @@ private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", - "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "engine_condition_pushdown", - "exists_to_in", "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", - "index_merge", "index_merge_intersection", "index_merge_sort_intersection", - "index_merge_sort_union", "index_merge_union", "in_to_exists", "join_cache_bka", - "join_cache_hashed", "join_cache_incremental", "loosescan", "materialization", "mrr", - "mrr_cost_based", "mrr_sort_keys", "not_null_range_scan", "optimize_join_buffer_size", - "orderby_uses_equalities", "outer_join_with_cache", "partial_match_rowid_merge", - "partial_match_table_scan", "rowid_filter", "semijoin", "semijoin_with_cache", "split_materialized", - "subquery_cache", "table_elimination" }; + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "exists_to_in", + "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", "index_merge", + "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", + "index_merge_union", "in_to_exists", "join_cache_bka", "join_cache_hashed", + "join_cache_incremental", "loosescan", "materialization", "mrr", "mrr_cost_based", "mrr_sort_keys", + "not_null_range_scan", "optimize_join_buffer_size", "orderby_uses_equalities", + "outer_join_with_cache", "partial_match_rowid_merge", "partial_match_table_scan", "rowid_filter", + "semijoin", "semijoin_with_cache", "split_materialized", "subquery_cache", "table_elimination" }; List optionSubset = Arrays.asList(Randomly.fromOptions(options)); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); @@ -191,9 +190,9 @@ public static SQLQueryAdapter resetOptimizer() { public static List getAllOptimizer(MariaDBGlobalState globalState) { List result = new ArrayList<>(); String[] options = { "condition_pushdown_for_derived", "condition_pushdown_for_subquery", - "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "engine_condition_pushdown", - "exists_to_in", "extended_keys", "firstmatch", "index_condition_pushdown", "hash_join_cardinality", - "index_merge", "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", + "condition_pushdown_from_having", "derived_merge", "derived_with_keys", "exists_to_in", "extended_keys", + "firstmatch", "index_condition_pushdown", "hash_join_cardinality", "index_merge", + "index_merge_intersection", "index_merge_sort_intersection", "index_merge_sort_union", "index_merge_union", "in_to_exists", "join_cache_bka", "join_cache_hashed", "join_cache_incremental", "loosescan", "materialization", "mrr", "mrr_cost_based", "mrr_sort_keys", "not_null_range_scan", "optimize_join_buffer_size", "orderby_uses_equalities", "outer_join_with_cache", diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 94db1fab7..79333eb36 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -135,12 +135,11 @@ private static String getOptimizerSwitchConfiguration(Randomly r) { StringBuilder sb = new StringBuilder(); sb.append("'"); String[] options = { "index_merge", "index_merge_union", "index_merge_sort_union", - "index_merge_intersection", "engine_condition_pushdown", "index_condition_pushdown", "mrr", - "mrr_cost_based", "block_nested_loop", "batched_key_access", "materialization", "semijoin", - "loosescan", "firstmatch", "duplicateweedout", "subquery_materialization_cost_based", - "use_index_extensions", "condition_fanout_filter", "derived_merge", "use_invisible_indexes", - "skip_scan", "hash_join", "subquery_to_derived", "prefer_ordering_index", - "derived_condition_pushdown" }; + "index_merge_intersection", "index_condition_pushdown", "mrr", "mrr_cost_based", + "block_nested_loop", "batched_key_access", "materialization", "semijoin", "loosescan", "firstmatch", + "duplicateweedout", "subquery_materialization_cost_based", "use_index_extensions", + "condition_fanout_filter", "derived_merge", "use_invisible_indexes", "skip_scan", "hash_join", + "subquery_to_derived", "prefer_ordering_index", "derived_condition_pushdown" }; List optionSubset = Randomly.nonEmptySubset(options); sb.append(optionSubset.stream().map(s -> s + "=" + Randomly.fromOptions("on", "off")) .collect(Collectors.joining(","))); From b7bbc1dc95e087267f28330cb4942ca48c1bcfe3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Thu, 11 Apr 2024 16:26:08 +0800 Subject: [PATCH 441/856] [MariaDB] Add key length restriction to expected errors --- src/sqlancer/mariadb/gen/MariaDBTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java index 4376b62db..23f09a420 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableGenerator.java @@ -142,6 +142,7 @@ private void createOrReplaceTable() { sb.append("IF NOT EXISTS "); } sb.append(tableName); + errors.add("Specified key was too long; max key length is"); } } From acd0b46bd04390190f717aa5ab1d6c4b7308a63b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 18 May 2024 09:55:28 +0800 Subject: [PATCH 442/856] [MariaDB] Add bug MDEV-33893 --- src/sqlancer/mariadb/MariaDBBugs.java | 3 +++ .../mariadb/gen/MariaDBTableAdminCommandGenerator.java | 5 ++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mariadb/MariaDBBugs.java b/src/sqlancer/mariadb/MariaDBBugs.java index fb1770763..5a1bfa5cc 100644 --- a/src/sqlancer/mariadb/MariaDBBugs.java +++ b/src/sqlancer/mariadb/MariaDBBugs.java @@ -26,6 +26,9 @@ public final class MariaDBBugs { // https://jira.mariadb.org/browse/MDEV-32143 public static boolean bug32143 = true; + // https://jira.mariadb.org/browse/MDEV-33893 + public static boolean bug33893 = true; + private MariaDBBugs() { } diff --git a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java index a141ca58d..dd40739f4 100644 --- a/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBTableAdminCommandGenerator.java @@ -8,6 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryResultCheckAdapter; +import sqlancer.mariadb.MariaDBBugs; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; @@ -54,7 +55,9 @@ public static SQLQueryAdapter checkTable(MariaDBSchema newSchema) { public static SQLQueryAdapter optimizeTable(MariaDBSchema newSchema) { StringBuilder sb = addCommandAndTables(newSchema, "OPTIMIZE TABLE"); - MariaDBCommon.addWaitClause(sb); + if (!MariaDBBugs.bug33893) { + MariaDBCommon.addWaitClause(sb); + } return checkForMsgText(sb, s -> s.equals("OK") || s.equals("Table does not support optimize, doing recreate + analyze instead") || s.contentEquals("Table is already up to date") || s.contains("Lock wait timeout") From eba08c4d11cb8a696c379c692ac879896123ab3a Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 21 May 2024 11:34:24 -0400 Subject: [PATCH 443/856] Upgrade CockroachDB to v24.1.0-rc.2 --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index dcbb1af99..89e447dc3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -138,11 +138,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.2.0-beta.1.linux-amd64.tgz | tar xvz - cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.1.0-rc.2.linux-amd64.tgz | tar xvz + cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test @@ -163,11 +163,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v23.2.0-beta.1.linux-amd64.tgz | tar xvz - cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.1.0-rc.2.linux-amd64.tgz | tar xvz + cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v23.2.0-beta.1.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test From 59470daff17e4de74079bce33dce3ef878970873 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 21 May 2024 13:07:29 -0400 Subject: [PATCH 444/856] Consolidate "unsupported binary operator" errors for CockroachDB --- src/sqlancer/cockroachdb/CockroachDBErrors.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 63d0b30ce..1f3e1a587 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -100,11 +100,7 @@ public static List getExpressionErrors() { errors.add("LOOKUP can only be used with INNER or LEFT joins"); // TODO errors.add("ambiguous binary operator: || "); - errors.add(" ERROR: unsupported binary operator: || (desired )"); - errors.add("unsupported binary operator: || (desired )"); - errors.add("incompatible value type: unsupported binary operator: || (desired )"); - errors.add("unsupported binary operator: || (desired )"); - errors.add("unsupported binary operator: || (desired )"); + errors.add("unsupported binary operator"); errors.add("parsing as type timestamp: empty or blank input"); errors.add("parsing as type timestamp: field"); errors.add("as type time"); @@ -121,7 +117,6 @@ public static List getExpressionErrors() { errors.add("numeric constant out of int64 range"); errors.add("unknown signature: overlay"); errors.add("unknown signature: substring"); - errors.add("unsupported binary operator: + (desired )"); errors.add("unsupported comparison operator"); errors.add("unknown signature: chr(decimal) (desired )"); errors.add("unknown signature: to_english(decimal) (desired )"); @@ -136,7 +131,6 @@ public static List getExpressionErrors() { errors.add("has type decimal"); errors.add("to be of type decimal, found type int"); errors.add("value type decimal doesn't match type int"); - errors.add("unsupported binary operator: / (desired )"); errors.add("(desired )"); errors.add("(desired )"); errors.add("(desired )"); From 3151b5c49392748b73c3e32700d8de816790b2ed Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Tue, 21 May 2024 13:09:43 -0400 Subject: [PATCH 445/856] Consolidate "could not parse JSON" errors for CockroachDB --- src/sqlancer/cockroachdb/CockroachDBErrors.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 1f3e1a587..b52df11bd 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -146,10 +146,7 @@ public static List getExpressionErrors() { errors.add("exists but is not a directory"); // TODO - errors.add("could not parse JSON: trailing characters after JSON document"); - errors.add("could not parse JSON: unable to decode JSON: invalid character"); - errors.add("could not parse JSON: unable to decode JSON: EOF"); - errors.add("could not parse JSON: unable to decode JSON: unexpected EOF"); + errors.add("could not parse JSON"); errors.add("can't order by column type jsonb"); errors.add("odd length hex string"); From 43a373a1feeac8a1f5cf5bfc205db05e71c2e302 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Sun, 16 Jun 2024 13:15:31 +0800 Subject: [PATCH 446/856] [Doris] Updated Doris to 2.1.4 (#949) * [Doris] Update deployment in CI * [Doris] Update group by columns to match fetch columns * [Doris] Prevent generation of null for some expressions * [Doris] Add bug 36070 * [Doris] Add new error message * [Doris] Update CI script and pin to version 2.1.3 * [Doris] Add bug 36072 * [Doris] Fix bugs in TLP Aggregate oracle * [Doris] Add bug 34342 * [Doris] Add bug 36343 * [Doris] Add bug 36346 * [Doris] Update bugs list * [Doris] Update to version 2.1.4 * [Doris] Refactor tests * [Doris] Add bug 36351 --- .github/workflows/main.yml | 30 +++++------ src/sqlancer/doris/DorisBugs.java | 48 ++++++++--------- src/sqlancer/doris/DorisErrors.java | 2 + src/sqlancer/doris/DorisSchema.java | 10 +++- .../gen/DorisNewExpressionGenerator.java | 39 +++++++++++--- ...DorisQueryPartitioningAggregateTester.java | 26 ++++++--- .../tlp/DorisQueryPartitioningBase.java | 7 +-- .../DorisQueryPartitioningGroupByTester.java | 2 +- .../DorisQueryPartitioningHavingTester.java | 1 + test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestDoris.java | 54 ------------------- test/sqlancer/dbms/TestDorisNoREC.java | 26 +++++++++ test/sqlancer/dbms/TestDorisPQS.java | 26 +++++++++ test/sqlancer/dbms/TestDorisTLP.java | 26 +++++++++ 14 files changed, 188 insertions(+), 110 deletions(-) delete mode 100644 test/sqlancer/dbms/TestDoris.java create mode 100644 test/sqlancer/dbms/TestDorisNoREC.java create mode 100644 test/sqlancer/dbms/TestDorisPQS.java create mode 100644 test/sqlancer/dbms/TestDorisTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 20a591cb8..9fb049f88 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -544,26 +544,26 @@ jobs: - name: Set up Apache Doris run: | sudo sysctl -w vm.max_map_count=2000000 - LASTEST_TAG=$(curl -s GET https://api.github.com/repos/apache/doris/releases | jq -r '.[].tag_name' | sed -n 1p) - LASTEST_TAG_BIG_VERSION=$(echo ${LASTEST_TAG} | awk '{split($1, arr, "."); print arr[1]"."arr[2]}') - curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz" - curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz" - curl -LJO "https://www.apache.org/dyn/mirrors/mirrors.cgi?action=download&filename=doris/${LASTEST_TAG_BIG_VERSION}/${LASTEST_TAG}/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz" - mkdir ./doris - tar xf apache-doris-fe-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-fe-${LASTEST_TAG}-bin-x86_64 doris/fe - tar xf apache-doris-be-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-be-${LASTEST_TAG}-bin-x86_64 doris/be - tar xf apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64.tar.xz -C ./doris && mv doris/apache-doris-dependencies-${LASTEST_TAG}-bin-x86_64 doris/dependencies - cp doris/dependencies/*.jar doris/be/lib/ - doris/fe/bin/start_fe.sh --daemon - doris/be/bin/start_be.sh --daemon - sudo apt install libnet-ifconfig-wrapper-perl --assume-yes - IP=$(ifconfig eth0 | grep inet | grep -v inet6 | awk '{print $2}') + wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz + tar zxf apache-doris-2.1.4-bin-x64.tar.gz + mv apache-doris-2.1.4-bin-x64 apache-doris + sudo swapoff -a + cd apache-doris/fe + ./bin/start_fe.sh --daemon + cd ../be + ./bin/start_be.sh --daemon + + sleep 30 + IP=$(hostname -I | awk '{print $1}') mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';" mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: DORIS_AVAILABLE=true mvn -Dtest=TestDoris test + run: | + DORIS_AVAILABLE=true mvn -Dtest=TestDorisNoREC test + DORIS_AVAILABLE=true mvn -Dtest=TestDorisPQS test + DORIS_AVAILABLE=true mvn -Dtest=TestDorisTLP test reducer: name: Reducer Tests diff --git a/src/sqlancer/doris/DorisBugs.java b/src/sqlancer/doris/DorisBugs.java index 080aa0450..956be1683 100644 --- a/src/sqlancer/doris/DorisBugs.java +++ b/src/sqlancer/doris/DorisBugs.java @@ -1,43 +1,43 @@ package sqlancer.doris; public final class DorisBugs { - // https://github.com/apache/doris/issues/17697 - // Logical bug about where true not in (columns) - public static boolean bug17697 = true; - - // https://github.com/apache/doris/issues/17700 - // Cannot use between and in boolean column - public static boolean bug17700 = true; - - // https://github.com/apache/doris/issues/17701 - // Wrong result of `where column not in (values)` - public static boolean bug17701 = true; - - // https://github.com/apache/doris/issues/17705 - // Different result caused by `where` split and union all - public static boolean bug17705 = true; - // https://github.com/apache/doris/issues/19370 // Internal Error occur in GroupBy&Having sql // fixed by https://github.com/apache/doris/pull/19559 - public static boolean bug19370 = true; + public static boolean bug19370; // https://github.com/apache/doris/issues/19374 // Different result of having not ($value in column) and having ($value not in column) // fixed by https://github.com/apache/doris/pull/19471 - public static boolean bug19374 = true; + public static boolean bug19374; // https://github.com/apache/doris/issues/19611 // ERROR occur in nested subqueries with same column name and union public static boolean bug19611 = true; - // https://github.com/apache/doris/issues/19613 - // Wrong result when right outer join and where false - public static boolean bug19613 = true; + // https://github.com/apache/doris/issues/36070 + // Expression evaluate to NULL but is treated as FALSE in where clause + public static boolean bug36070 = true; + + // https://github.com/apache/doris/issues/36072 + // SELECT DISTINCT does not work with aggregate key column + public static boolean bug36072 = true; + + // https://github.com/apache/doris/issues/36342 + // Wrong result with INNER JOIN and CURRENT_TIMESTAMP + public static boolean bug36342 = true; + + // https://github.com/apache/doris/issues/36343 + // Wrong result with SELECT DISTINCT and UNIQUE model + public static boolean bug36343 = true; + + // https://github.com/apache/doris/issues/36346 + // Wrong result with LEFT JOIN SELECT DISTINCT and IN operation + public static boolean bug36346 = true; - // https://github.com/apache/doris/issues/19614 - // Wrong result when value like column from table_join - public static boolean bug19614 = true; + // https://github.com/apache/doris/issues/36351 + // Wrong result with TINYINT column with value -1049190528 + public static boolean bug36351 = true; private DorisBugs() { diff --git a/src/sqlancer/doris/DorisErrors.java b/src/sqlancer/doris/DorisErrors.java index 8e6fdfdc9..28e93b80a 100644 --- a/src/sqlancer/doris/DorisErrors.java +++ b/src/sqlancer/doris/DorisErrors.java @@ -57,6 +57,7 @@ public static List getExpressionErrors() { if (DorisBugs.bug19611) { errors.add("Duplicated inline view column alias"); } + errors.add("Arithmetic overflow"); return errors; } @@ -72,6 +73,7 @@ public static List getInsertErrors() { errors.add("Only value columns of unique table could be updated"); errors.add("Only unique olap table could be updated"); errors.add("Number out of range"); + errors.add("Arithmetic overflow"); return errors; } diff --git a/src/sqlancer/doris/DorisSchema.java b/src/sqlancer/doris/DorisSchema.java index d452f784a..76697fad7 100644 --- a/src/sqlancer/doris/DorisSchema.java +++ b/src/sqlancer/doris/DorisSchema.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -30,7 +31,14 @@ public enum DorisTableDataModel { UNIQUE, AGGREGATE, DUPLICATE; public static DorisTableDataModel getRandom() { - return Randomly.fromOptions(values()); + List validOptions = new ArrayList<>(Arrays.asList(values())); + if (DorisBugs.bug36072) { + validOptions.remove(AGGREGATE); + } + if (DorisBugs.bug36343) { + validOptions.remove(UNIQUE); + } + return Randomly.fromList(validOptions); } } diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java index 5dcb82e8f..f283d0c39 100644 --- a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -7,10 +7,12 @@ import java.util.Set; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.doris.DorisBugs; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisDataType; @@ -117,10 +119,12 @@ public DorisExpression generateExpression(DorisDataType type, int depth) { return function.getCall(type, this, depth + 1); } } - if (globalState.getDbmsSpecificOptions().testCasts && Randomly.getBooleanWithRatherLowProbability()) { + if (!DorisBugs.bug36070 && type != DorisDataType.NULL && globalState.getDbmsSpecificOptions().testCasts + && Randomly.getBooleanWithRatherLowProbability()) { return new DorisCastOperation(DorisExprToNode.cast(generateExpression(getRandomType(), depth + 1)), type); } - if (globalState.getDbmsSpecificOptions().testCase && Randomly.getBooleanWithRatherLowProbability()) { + if (!DorisBugs.bug36070 && globalState.getDbmsSpecificOptions().testCase + && Randomly.getBooleanWithRatherLowProbability()) { DorisExpression expr = generateExpression(DorisDataType.BOOLEAN, depth + 1); List conditions = new ArrayList<>(); List cases = new ArrayList<>(); @@ -192,7 +196,7 @@ DorisExpression generateBooleanExpression(int depth) { allowAggregateFunctions = false; } List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); - if (!globalState.getDbmsSpecificOptions().testIn) { + if (DorisBugs.bug36346 || !globalState.getDbmsSpecificOptions().testIn) { validOptions.remove(BooleanExpression.IN_OPERATION); } if (!globalState.getDbmsSpecificOptions().testBinaryLogicals) { @@ -201,7 +205,7 @@ DorisExpression generateBooleanExpression(int depth) { if (!globalState.getDbmsSpecificOptions().testBinaryComparisons) { validOptions.remove(BooleanExpression.BINARY_COMPARISON); } - if (!globalState.getDbmsSpecificOptions().testBetween) { + if (DorisBugs.bug36070 || !globalState.getDbmsSpecificOptions().testBetween) { validOptions.remove(BooleanExpression.BETWEEN); } @@ -306,8 +310,11 @@ public DorisExpression isNull(DorisExpression predicate) { } public DorisExpression generateConstant(DorisDataType type, boolean isNullable) { - if (isNullable && Randomly.getBooleanWithSmallProbability()) { - createConstant(DorisDataType.NULL); + if (!isNullable) { + return createConstantWithoutNull(type); + } + if (Randomly.getBooleanWithSmallProbability()) { + return createConstant(DorisDataType.NULL); } return createConstant(type); } @@ -320,12 +327,29 @@ public DorisExpression generateConstant(DorisDataType type) { return createConstant(type); } + public DorisExpression createConstantWithoutNull(DorisDataType type) { + DorisExpression constant = createConstant(type); + int loopCount = 0; + while (constant instanceof DorisConstant.DorisNullConstant && loopCount < 1000) { + constant = createConstant(type); + loopCount++; + } + if (constant instanceof DorisConstant.DorisNullConstant) { + throw new IgnoreMeException(); + } + return constant; + } + public DorisExpression createConstant(DorisDataType type) { Randomly r = globalState.getRandomly(); long timestamp; switch (type) { case INT: if (globalState.getDbmsSpecificOptions().testIntConstants) { + long number = r.getInteger(); + if (DorisBugs.bug36351 && number == -1049190528) { + number = 0; + } return DorisConstant.createIntConstant(r.getInteger()); } return DorisConstant.createNullConstant(); @@ -368,6 +392,9 @@ public DorisExpression createConstant(DorisDataType type) { if (globalState.getDbmsSpecificOptions().testDateTimeConstants) { // [1970-01-01 08:00:00, 3000-01-01 00:00:00] timestamp = globalState.getRandomly().getLong(0, 32503651200L); + if (DorisBugs.bug36342) { + return DorisConstant.createDatetimeConstant(timestamp); + } return Randomly.fromOptions(DorisConstant.createDatetimeConstant(timestamp), DorisConstant.createDatetimeConstant()); } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java index d8076d3e5..1f3fe66f1 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -74,9 +74,21 @@ public void check() throws SQLException { state.getState().getLocalState().log( "--" + originalQuery + ";\n--" + metamorphicQuery + "\n-- " + firstResult + "\n-- " + secondResult); - if (firstResult == null && secondResult != null - || firstResult != null && (!firstResult.contentEquals(secondResult) - && !ComparatorHelper.isEqualDouble(firstResult, secondResult))) { + if (firstResult == null && secondResult == null) { + return; + } + if (firstResult == null) { + throw new AssertionError(); + } + firstResult = firstResult.replace("\0", ""); + if (firstResult.contentEquals("0") && secondResult == null) { + return; + } + if (secondResult == null) { + throw new AssertionError(); + } + secondResult = secondResult.replace("\0", ""); + if (!firstResult.contentEquals(secondResult) && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { throw new AssertionError(); } @@ -94,6 +106,11 @@ private String createMetamorphicUnionQuery(DorisSelect select, DorisSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DorisSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DorisSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); + if (Randomly.getBooleanWithSmallProbability()) { + leftSelect.setGroupByExpressions(groupByExpression); + middleSelect.setGroupByExpressions(groupByExpression); + rightSelect.setGroupByExpressions(groupByExpression); + } metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; metamorphicQuery += DorisToStringVisitor.asString(leftSelect) + " UNION ALL " + DorisToStringVisitor.asString(middleSelect) + " UNION ALL " @@ -185,9 +202,6 @@ private DorisSelect getSelect(List> aggregates, List allColumnValues = targetTables.getColumns().stream() + .map(c -> new DorisColumnValue(c, null)).collect(Collectors.toList()); HashSet columnOfLeafNode = new HashSet<>(); gen.setColumnOfLeafNode(columnOfLeafNode); initializeTernaryPredicateVariants(); select = new DorisSelect(); - columnOfLeafNode.addAll(targetTables.getColumns().stream().map(c -> new DorisColumnValue(c, null)) - .collect(Collectors.toList())); - groupByExpression = new ArrayList<>(columnOfLeafNode); + columnOfLeafNode.addAll(allColumnValues); + groupByExpression = new ArrayList<>(allColumnValues); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); List> tableList = tables.stream() diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java index 00a0cf28a..8e173d7ae 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java @@ -27,7 +27,7 @@ public DorisQueryPartitioningGroupByTester(DorisGlobalState state) { @Override public void check() throws SQLException { super.check(); - select.setGroupByExpressions(groupByExpression); + select.setGroupByExpressions(select.getFetchColumns()); select.setWhereClause(null); String originalQueryString = DorisToStringVisitor.asString(select); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java index 676065616..b3398f229 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -31,6 +31,7 @@ public void check() throws SQLException { if (Randomly.getBoolean()) { select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); } + select.setFetchColumns(groupByExpression); boolean orderBy = Randomly.getBoolean(); if (orderBy) { List> constants = new ArrayList<>(); diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 8cea624c7..baab98225 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -4,6 +4,7 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { diff --git a/test/sqlancer/dbms/TestDoris.java b/test/sqlancer/dbms/TestDoris.java deleted file mode 100644 index fd3c1e95a..000000000 --- a/test/sqlancer/dbms/TestDoris.java +++ /dev/null @@ -1,54 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; -import sqlancer.Randomly; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestDoris { - private final String host = "127.0.0.1"; - private final String port = "9030"; - private final String username = "sqlancer"; - private final String password = "sqlancer"; - - @Test - public void testdorisNoREC() { - String dorisAvailable = System.getenv("DORIS_AVAILABLE"); - boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); - assumeTrue(dorisIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--username", username, "--password", password, "--host", host, "--port", port, "doris", - "--oracle", "NOREC")); - } - - @Test - public void testdorisPQS() { - String dorisAvailable = System.getenv("DORIS_AVAILABLE"); - boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); - assumeTrue(dorisIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--username", username, "--password", password, "--host", host, "--port", port, "doris", - "--oracle", "PQS")); - } - - @Test - public void testdorisTLPQueryPartitioning() { - String dorisAvailable = System.getenv("DORIS_AVAILABLE"); - boolean dorisIsAvailable = dorisAvailable != null && dorisAvailable.equalsIgnoreCase("true"); - assumeTrue(dorisIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--username", username, "--password", password, "--host", host, "--port", port, "doris", - "--oracle", "QUERY_PARTITIONING")); - } -} diff --git a/test/sqlancer/dbms/TestDorisNoREC.java b/test/sqlancer/dbms/TestDorisNoREC.java new file mode 100644 index 000000000..76a9d50a8 --- /dev/null +++ b/test/sqlancer/dbms/TestDorisNoREC.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDorisNoREC { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DORIS_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "NOREC")); + } +} diff --git a/test/sqlancer/dbms/TestDorisPQS.java b/test/sqlancer/dbms/TestDorisPQS.java new file mode 100644 index 000000000..5760003d3 --- /dev/null +++ b/test/sqlancer/dbms/TestDorisPQS.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDorisPQS { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DORIS_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "PQS")); + } +} diff --git a/test/sqlancer/dbms/TestDorisTLP.java b/test/sqlancer/dbms/TestDorisTLP.java new file mode 100644 index 000000000..f8bea9e47 --- /dev/null +++ b/test/sqlancer/dbms/TestDorisTLP.java @@ -0,0 +1,26 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDorisTLP { + private final String host = "127.0.0.1"; + private final String port = "9030"; + private final String username = "sqlancer"; + private final String password = "sqlancer"; + + @Test + public void testdorisTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DORIS_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "doris", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--username", username, "--password", password, "--host", host, "--port", port, "doris", + "--oracle", "QUERY_PARTITIONING")); + } +} From ad5d891bf683c24c760ae9638cd337751d6d4ce4 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Sun, 16 Jun 2024 14:28:24 +0800 Subject: [PATCH 447/856] [Databend] Pinned version to v1.2.452 (#946) * [Databend] Update error message for violation of not-null constraint * [Databend] Add expected error for table generation * [Databend] Add bug 15568 * [Databend] Add bug 15569 * [Databend] Add bug 15570 * [Databend] Add error messages returned from new constraints * [Databend] Add bug 15572 * [Databend] Pin CI to v1.2.452 --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/DatabendBugs.java | 4 ++++ src/sqlancer/databend/DatabendErrors.java | 8 +++++++- src/sqlancer/databend/DatabendProvider.java | 7 +++++++ .../databend/gen/DatabendNewExpressionGenerator.java | 10 ++++++++++ src/sqlancer/databend/gen/DatabendTableGenerator.java | 2 ++ 6 files changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9fb049f88..8e5269fb0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -176,7 +176,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend + image: datafuselabs/databend:v1.2.452 env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index eec2c39d0..dd11512d8 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -15,6 +15,10 @@ public final class DatabendBugs { public static boolean bug9236 = true; // https://github.com/datafuselabs/databend/issues/9236 public static boolean bug9264 = true; // https://github.com/datafuselabs/databend/issues/9264 public static boolean bug9806 = true; // https://github.com/datafuselabs/databend/issues/9806 + public static boolean bug15568 = true; // https://github.com/datafuselabs/databend/issues/15568 + public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 + public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 + public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 2f588480f..fdd8a3a69 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -21,6 +21,8 @@ public static List getExpressionErrors() { errors.add("no overload satisfies `not(Float64 NULL)`"); // TODO databend不允许出现not(float)，而a/b为float errors.add("no overload satisfies `not(Float64)`"); errors.add("number overflowed while evaluating function"); // 表达式数值溢出 + errors.add("Unable to get field named"); + errors.add("no overload satisfies `and_filters"); if (DatabendBugs.bug9162) { errors.add("downcast column error"); } @@ -42,6 +44,9 @@ public static List getExpressionErrors() { if (DatabendBugs.bug9806) { errors.add("segment pruning failure"); } + if (DatabendBugs.bug15568) { + errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE @@ -63,7 +68,7 @@ public static List getInsertErrors() { errors.add("Division by zero"); errors.add("/ by zero"); - errors.add("Can't cast column from null into non-nullable type"); + errors.add("violates not-null constraint"); errors.add("number overflowed while evaluating function `"); // 不能在int16类型column上插入int64的数据 return errors; @@ -80,6 +85,7 @@ public static List getGroupByErrors() { errors.add("/ by zero"); errors.add("Can't cast column from null into non-nullable type"); errors.add("GROUP BY position"); + errors.add("GROUP BY items can't contain aggregate functions or window functions"); return errors; } diff --git a/src/sqlancer/databend/DatabendProvider.java b/src/sqlancer/databend/DatabendProvider.java index 0ab0d4c9b..df7802027 100644 --- a/src/sqlancer/databend/DatabendProvider.java +++ b/src/sqlancer/databend/DatabendProvider.java @@ -133,6 +133,13 @@ public SQLConnection createDatabase(DatabendGlobalState globalState) throws SQLE s.execute("USE " + databaseName); globalState.getState().logStatement("USE " + databaseName); } + if (DatabendBugs.bug15569) { + con.close(); + String urlWithRetry = String.format( + "jdbc:mysql://%s:%d/%s?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true&autoReconnect=true", + host, port, databaseName); + con = DriverManager.getConnection(urlWithRetry, username, password); + } return new SQLConnection(con); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index feb7b3483..2faf4376b 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -11,6 +11,7 @@ import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; @@ -160,6 +161,15 @@ DatabendExpression generateBooleanExpression(int depth) { allowAggregateFunctions = false; } List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); + if (DatabendBugs.bug15570) { + validOptions.remove(BooleanExpression.LIKE); + validOptions.remove(BooleanExpression.IN_OPERATION); + validOptions.remove(BooleanExpression.BETWEEN); + validOptions.remove(BooleanExpression.BINARY_COMPARISON); + } + if (DatabendBugs.bug15572) { + validOptions.remove(BooleanExpression.NOT); + } BooleanExpression option = Randomly.fromList(validOptions); switch (option) { case POSTFIX_OPERATOR: diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index a34279fd2..ab4900c07 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -19,6 +20,7 @@ public class DatabendTableGenerator { public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); + DatabendErrors.addExpressionErrors(errors); StringBuilder sb = new StringBuilder(); String tableName = globalState.getSchema().getFreeTableName(); sb.append("CREATE TABLE "); From 3989d762ede46b8251a104fa66a294aa27897271 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 29 Jun 2024 16:27:43 +0800 Subject: [PATCH 448/856] [Databend] Refactor tests --- .github/workflows/main.yml | 4 +- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestDatabend.java | 108 ---------------------- test/sqlancer/dbms/TestDatabendNoREC.java | 23 +++++ test/sqlancer/dbms/TestDatabendPQS.java | 23 +++++ test/sqlancer/dbms/TestDatabendTLP.java | 21 +++++ 6 files changed, 71 insertions(+), 109 deletions(-) delete mode 100644 test/sqlancer/dbms/TestDatabend.java create mode 100644 test/sqlancer/dbms/TestDatabendNoREC.java create mode 100644 test/sqlancer/dbms/TestDatabendPQS.java create mode 100644 test/sqlancer/dbms/TestDatabendTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e5269fb0..6b920cd61 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -197,7 +197,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendTLP test + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendNoREC test + DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendPQS test duckdb: name: DBMS Tests (DuckDB) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index baab98225..67c406fba 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -4,6 +4,7 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestDatabend.java b/test/sqlancer/dbms/TestDatabend.java deleted file mode 100644 index 707d7b8a9..000000000 --- a/test/sqlancer/dbms/TestDatabend.java +++ /dev/null @@ -1,108 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; -import sqlancer.Randomly; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestDatabend { - - @Test - public void testDatabendNoREC() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); - } - - @Test - public void testDatabendPQS() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); - } - - @Test - public void testDatabendTLPQueryPartitioning() { - String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - assumeTrue(databendIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); - } - - // @Test - // public void testDatabendTLPWhere() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "WHERE")); - // } - // - // @Test - // public void testDatabendTLPGroupBy() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "GROUP_BY")); - // } - // - // @Test - // public void testDatabendTLPHaving() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "HAVING")); - // } - // - // @Test - // public void testDatabendTLPDistinct() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "DISTINCT")); - // } - // - // @Test - // public void testDatabendTLPAggregate() { - // String databendAvailable = System.getenv("DATABEND_AVAILABLE"); - // boolean databendIsAvailable = databendAvailable != null && databendAvailable.equalsIgnoreCase("true"); - // assumeTrue(databendIsAvailable); - // assertEquals(0, - // Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", - // "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", - // "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), - // "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "AGGREGATE")); - // } - -} diff --git a/test/sqlancer/dbms/TestDatabendNoREC.java b/test/sqlancer/dbms/TestDatabendNoREC.java new file mode 100644 index 000000000..679f8c161 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendNoREC.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.Randomly; + +public class TestDatabendNoREC { + + @Test + public void testDatabendNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "NOREC")); + } + +} diff --git a/test/sqlancer/dbms/TestDatabendPQS.java b/test/sqlancer/dbms/TestDatabendPQS.java new file mode 100644 index 000000000..fba733d44 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendPQS.java @@ -0,0 +1,23 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.Randomly; + +public class TestDatabendPQS { + + @Test + public void testDatabendPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "PQS")); + } + +} diff --git a/test/sqlancer/dbms/TestDatabendTLP.java b/test/sqlancer/dbms/TestDatabendTLP.java new file mode 100644 index 000000000..27ba53416 --- /dev/null +++ b/test/sqlancer/dbms/TestDatabendTLP.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; +import sqlancer.Randomly; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestDatabendTLP { + + @Test + public void testDatabendTLPQueryPartitioning() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATABEND_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", + "--num-queries", TestConfig.NUM_QUERIES, "--database-prefix", "databend", + "--random-string-generation", String.valueOf(Randomly.StringGenerationStrategy.ALPHANUMERIC), + "--host", "127.0.0.1", "--port", "3307", "databend", "--oracle", "QUERY_PARTITIONING")); + } +} From 37de4a2cf8b90ce7c22f3db37d85590d6dca873f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 29 Jun 2024 16:27:54 +0800 Subject: [PATCH 449/856] [Databend] Update to v1.2.542 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6b920cd61..10cd3ace5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -176,7 +176,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.452 + image: datafuselabs/databend:v1.2.542-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer From 92fb68865daa18d1ca6e99ef7da1da6614a0abe3 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Mon, 8 Jul 2024 14:42:43 +0800 Subject: [PATCH 450/856] Remove duplicated logs in NoREC oracle (#953) --- src/sqlancer/common/oracle/NoRECOracle.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index acc7cffac..530488296 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -123,10 +123,6 @@ public Reproducer getLastReproducer() { private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(queryString); - } - int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { @@ -151,10 +147,6 @@ private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState< private int extractCounts(String queryString, ExpectedErrors errors, SQLGlobalState state) { SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(queryString); - } - int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { From b41b54ea82a9a0a24c6cc651c005de160389da34 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Sun, 7 Jul 2024 22:48:37 +0800 Subject: [PATCH 451/856] Initial DataFusion implementation --- .github/workflows/main.yml | 28 ++ README.md | 17 +- pom.xml | 23 + src/check_names.py | 1 + src/sqlancer/datafusion/DataFusionErrors.java | 44 ++ .../datafusion/DataFusionOptions.java | 43 ++ .../datafusion/DataFusionProvider.java | 135 +++++ src/sqlancer/datafusion/DataFusionSchema.java | 196 ++++++++ .../datafusion/DataFusionToStringVisitor.java | 99 ++++ src/sqlancer/datafusion/DataFusionUtil.java | 193 ++++++++ .../datafusion/ast/DataFusionConstant.java | 99 ++++ .../datafusion/ast/DataFusionExpression.java | 5 + .../datafusion/ast/DataFusionJoin.java | 92 ++++ .../datafusion/ast/DataFusionSelect.java | 67 +++ .../datafusion/gen/DataFusionBaseExpr.java | 258 ++++++++++ .../gen/DataFusionBaseExprFactory.java | 391 +++++++++++++++ .../gen/DataFusionExpressionGenerator.java | 227 +++++++++ .../gen/DataFusionInsertGenerator.java | 54 ++ .../gen/DataFusionTableGenerator.java | 33 ++ .../server/datafusion_server/Cargo.toml | 49 ++ .../server/datafusion_server/src/main.rs | 463 ++++++++++++++++++ .../test/DataFusionNoRECOracle.java | 95 ++++ .../test/DataFusionQueryPartitioningBase.java | 38 ++ ...ataFusionQueryPartitioningWhereTester.java | 67 +++ test/sqlancer/dbms/TestDataFusion.java | 21 + 25 files changed, 2730 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/datafusion/DataFusionErrors.java create mode 100644 src/sqlancer/datafusion/DataFusionOptions.java create mode 100644 src/sqlancer/datafusion/DataFusionProvider.java create mode 100644 src/sqlancer/datafusion/DataFusionSchema.java create mode 100644 src/sqlancer/datafusion/DataFusionToStringVisitor.java create mode 100644 src/sqlancer/datafusion/DataFusionUtil.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionConstant.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionExpression.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionJoin.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionSelect.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionBaseExpr.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java create mode 100644 src/sqlancer/datafusion/gen/DataFusionTableGenerator.java create mode 100644 src/sqlancer/datafusion/server/datafusion_server/Cargo.toml create mode 100644 src/sqlancer/datafusion/server/datafusion_server/src/main.rs create mode 100644 src/sqlancer/datafusion/test/DataFusionNoRECOracle.java create mode 100644 src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java create mode 100644 src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java create mode 100644 test/sqlancer/dbms/TestDataFusion.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e5269fb0..0f2a12c62 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -199,6 +199,34 @@ jobs: run: | DATABEND_AVAILABLE=true mvn -Dtest=TestDatabend test + datafusion: + name: DBMS Tests (DataFusion) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Run DataFusion Server + run: | + cd src/sqlancer/datafusion/server/datafusion_server + cargo run --features "datafusion_stable" & sleep 300 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + DATAFUSION_AVAILABLE=true mvn test -Pdatafusion-tests + duckdb: name: DBMS Tests (DuckDB) runs-on: ubuntu-latest diff --git a/README.md b/README.md index 83e207485..bb80d0794 100644 --- a/README.md +++ b/README.md @@ -56,11 +56,11 @@ Please find the `.bib` entries [here](docs/PAPERS.md). Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. | DBMS | Status | Expression Generation | Description | -|------------------------------|-------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ---------------------------- | ----------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | | MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | | PostgreSQL | Working | Typed | | -| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | +| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | | MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | | CockroachDB | Working | Typed | | | TiDB | Working | Untyped | | @@ -69,12 +69,13 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | | OceanBase | Working | Untyped | | | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | -| Databend | Working | Typed | | -| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | -| CnosDB |Working | Typed | The implementation of CnosDB currently uses Restful API. | -| Materialize |Working | Typed | -| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. -| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. +| Databend | Working | Typed | | +| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | +| CnosDB | Working | Typed | The implementation of CnosDB currently uses Restful API. | +| Materialize | Working | Typed | | +| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | +| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | +| DataFusion | Preliminary | Typed | Only basic SQL features are supported. | diff --git a/pom.xml b/pom.xml index 2b34d6c27..9bee83485 100644 --- a/pom.xml +++ b/pom.xml @@ -373,6 +373,11 @@ jsqlparser 4.6 + + org.apache.arrow + flight-sql-jdbc-driver + 16.1.0 + @@ -471,5 +476,23 @@ + + datafusion-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + 3.3.0 + + + **/TestDataFusion.java + + --add-opens java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED + + + + + diff --git a/src/check_names.py b/src/check_names.py index 63c1f093f..41694e060 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -38,6 +38,7 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["CnosDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cnosdb")) name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) + name_to_files["DataFusion"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "datafusion")) name_to_files["DuckDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "duckdb")) name_to_files["H2"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "h2")) name_to_files["HSQLDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "hsqldb")) diff --git a/src/sqlancer/datafusion/DataFusionErrors.java b/src/sqlancer/datafusion/DataFusionErrors.java new file mode 100644 index 000000000..d4b208f5b --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionErrors.java @@ -0,0 +1,44 @@ +package sqlancer.datafusion; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; + +import sqlancer.common.query.ExpectedErrors; + +public final class DataFusionErrors { + private DataFusionErrors() { + dfAssert(false, "Utility class cannot be instantiated"); + } + + /* + * During Oracle Checks, if ANY query returns one of the following error Then the current oracle check will be + * skipped. e.g.: NoREC Q1 -> throw an expected error NoREC Q2 -> succeed Since it's a known error, `SQLancer` will + * skip this check and don't report bug. + * + * Note now it's implemented this way for simplicity This way might cause false negative, because Q1 and Q2 should + * both succeed or both fail TODO(datafusion): ensure both succeed or both fail + */ + public static void registerExpectedExecutionErrors(ExpectedErrors errors) { + /* + * Expected + */ + errors.add("Error building plan"); // Randomly generated SQL is not valid and caused palning error + errors.add("Error during planning"); + errors.add("Execution error"); + errors.add("Overflow happened"); + errors.add("overflow"); + errors.add("Unsupported data type"); + errors.add("Divide by zero"); + /* + * Known bugs + */ + errors.add("to type Int64"); // https://github.com/apache/datafusion/issues/11252 + errors.add("bitwise"); // https://github.com/apache/datafusion/issues/11260 + errors.add("NestedLoopJoinExec"); // https://github.com/apache/datafusion/issues/11269 + /* + * False positives + */ + errors.add("Physical plan does not support logical expression AggregateFunction"); // False positive: when aggr + // is generated in where + // clause + } +} diff --git a/src/sqlancer/datafusion/DataFusionOptions.java b/src/sqlancer/datafusion/DataFusionOptions.java new file mode 100644 index 000000000..fcb0221a6 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionOptions.java @@ -0,0 +1,43 @@ +package sqlancer.datafusion; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.datafusion.DataFusionOptions.DataFusionOracleFactory; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.test.DataFusionNoRECOracle; +import sqlancer.datafusion.test.DataFusionQueryPartitioningWhereTester; + +@Parameters(commandDescription = "DataFusion") +public class DataFusionOptions implements DBMSSpecificOptions { + @Parameter(names = "--debug-info", description = "Show debug messages related to DataFusion", arity = 0) + public boolean showDebugInfo; + + @Override + public List getTestOracleFactory() { + return Arrays.asList(DataFusionOracleFactory.NOREC, DataFusionOracleFactory.QUERY_PARTITIONING_WHERE); + } + + public enum DataFusionOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DataFusionGlobalState globalState) throws SQLException { + return new DataFusionNoRECOracle(globalState); + } + }, + QUERY_PARTITIONING_WHERE { + @Override + public TestOracle create(DataFusionGlobalState globalState) throws SQLException { + return new DataFusionQueryPartitioningWhereTester(globalState); + } + } + } + +} diff --git a/src/sqlancer/datafusion/DataFusionProvider.java b/src/sqlancer/datafusion/DataFusionProvider.java new file mode 100644 index 000000000..161c324dd --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionProvider.java @@ -0,0 +1,135 @@ +package sqlancer.datafusion; + +import static java.lang.System.exit; +import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.DML; +import static sqlancer.datafusion.DataFusionUtil.displayTables; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.List; +import java.util.Properties; +import java.util.stream.Collectors; + +import com.google.auto.service.AutoService; + +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.SQLProviderAdapter; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionUtil.DataFusionInstanceID; +import sqlancer.datafusion.DataFusionUtil.DataFusionLogger; +import sqlancer.datafusion.gen.DataFusionInsertGenerator; +import sqlancer.datafusion.gen.DataFusionTableGenerator; + +@AutoService(DatabaseProvider.class) +public class DataFusionProvider extends SQLProviderAdapter { + + public DataFusionProvider() { + super(DataFusionGlobalState.class, DataFusionOptions.class); + } + + @Override + public void generateDatabase(DataFusionGlobalState globalState) throws Exception { + int tableCount = Randomly.fromOptions(1, 2, 3, 4, 5, 6, 7); + for (int i = 0; i < tableCount; i++) { + SQLQueryAdapter queryCreateRandomTable = new DataFusionTableGenerator().getQuery(globalState); + queryCreateRandomTable.execute(globalState); + globalState.updateSchema(); + globalState.dfLogger.appendToLog(DML, queryCreateRandomTable.toString() + "\n"); + } + + // Now only `INSERT` DML is supported + // If more DMLs are added later, should use`StatementExecutor` instead + // (see DuckDB's implementation for reference) + + globalState.updateSchema(); + List allTables = globalState.getSchema().getDatabaseTables(); + List allTablesName = allTables.stream().map(t -> t.getName()).collect(Collectors.toList()); + if (allTablesName.isEmpty()) { + System.out.println("Generate database failed"); + exit(1); + } + + // Randomly insert some data into existing tables + for (DataFusionTable table : allTables) { + int nInsertQuery = globalState.getRandomly().getInteger(0, 8); // [0, 10) + + for (int i = 0; i < nInsertQuery; i++) { + SQLQueryAdapter insertQuery = null; + try { + insertQuery = DataFusionInsertGenerator.getQuery(globalState, table); + } catch (IgnoreMeException e) { + // Only for special case: table has 0 column + continue; + } + + insertQuery.execute(globalState); + globalState.dfLogger.appendToLog(DML, insertQuery.toString() + "\n"); + } + } + + // TODO(datafusion) add `DataFUsionLogType.STATE` for this whole db state log + if (globalState.getDbmsSpecificOptions().showDebugInfo) { + System.out.println(displayTables(globalState, allTablesName)); + } + } + + @Override + public SQLConnection createDatabase(DataFusionGlobalState globalState) throws SQLException { + if (globalState.getDbmsSpecificOptions().showDebugInfo) { + System.out.println("A new database get created!\n"); + } + Properties props = new Properties(); + props.setProperty("UseEncryption", "false"); + // must set 'user' and 'password' to trigger server 'do_handshake()' + props.setProperty("user", "foo"); + props.setProperty("password", "bar"); + props.setProperty("create", globalState.getDatabaseName()); // Hack: use this property to let DataFusion server + // clear the current context + String url = "jdbc:arrow-flight-sql://127.0.0.1:50051"; + Connection connection = DriverManager.getConnection(url, props); + + return new SQLConnection(connection); + } + + @Override + public String getDBMSName() { + return "datafusion"; + } + + // If run SQLancer with multiple thread + // Each thread's instance will have its own `DataFusionGlobalState` + // It will store global states including: + // JDBC connection to DataFusion server + // Logger for this thread + public static class DataFusionGlobalState extends SQLGlobalState { + public DataFusionLogger dfLogger; + DataFusionInstanceID id; + + public DataFusionGlobalState() { + // HACK: test will only run in spawned thread, not main thread + // this way redundant logger files won't be created + if (Thread.currentThread().getName().equals("main")) { + return; + } + + id = new DataFusionInstanceID(Thread.currentThread().getName()); + try { + dfLogger = new DataFusionLogger(this, id); + } catch (Exception e) { + throw new IgnoreMeException(); + } + } + + @Override + protected DataFusionSchema readSchema() throws SQLException { + return DataFusionSchema.fromConnection(getConnection(), getDatabaseName()); + } + } +} diff --git a/src/sqlancer/datafusion/DataFusionSchema.java b/src/sqlancer/datafusion/DataFusionSchema.java new file mode 100644 index 000000000..4a5ed803a --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionSchema.java @@ -0,0 +1,196 @@ +package sqlancer.datafusion; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.ast.DataFusionConstant; +import sqlancer.datafusion.ast.DataFusionExpression; + +public class DataFusionSchema extends AbstractSchema { + + public DataFusionSchema(List databaseTables) { + super(databaseTables); + } + + // update existing tables in DB by query again + // (like `show tables;`) + public static DataFusionSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + DataFusionTable t = new DataFusionTable(tableName, databaseColumns, isView); + for (DataFusionColumn c : databaseColumns) { + c.setTable(t); + } + + databaseTables.add(t); + } + + return new DataFusionSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery("select table_name " + "from information_schema.tables " + + "where table_schema='public'" + "order by table_name; ")) { + while (rs.next()) { + tableNames.add(rs.getString(1)); + } + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery( + String.format("select * from information_schema.columns where table_name = '%s';", tableName))) { + while (rs.next()) { + String columnName = rs.getString("column_name"); + String dataType = rs.getString("data_type"); + boolean isNullable = rs.getString("is_nullable").contentEquals("YES"); + + DataFusionColumn c = new DataFusionColumn(columnName, + DataFusionDataType.parseFromDataFusionCatalog(dataType), isNullable); + columns.add(c); + } + } + } + + return columns; + } + + /* + * When adding a new type: 1. Update all methods inside this enum 2. Update all `DataFusionBaseExpr`'s signature, if + * it can support new type (in `DataFusionBaseExprFactory.java` + * + * Types are 'SQL DataType' in DataFusion's documentation + * https://datafusion.apache.org/user-guide/sql/data_types.html + */ + public enum DataFusionDataType { + + BIGINT, DOUBLE, BOOLEAN, NULL; + + public static DataFusionDataType getRandomWithoutNull() { + DataFusionDataType dt; + do { + dt = Randomly.fromOptions(values()); + } while (dt == DataFusionDataType.NULL); + return dt; + } + + // How to parse type in DataFusion's catalog to `DataFusionDataType` + // As displayed in: + // create table t1(v1 int, v2 bigint); + // select table_name, column_name, data_type from information_schema.columns; + public static DataFusionDataType parseFromDataFusionCatalog(String typeString) { + switch (typeString) { + case "Int64": + return DataFusionDataType.BIGINT; + case "Float64": + return DataFusionDataType.DOUBLE; + case "Boolean": + return DataFusionDataType.BOOLEAN; + default: + dfAssert(false, "Unreachable. All branches should be eovered"); + } + + dfAssert(false, "Unreachable. All branches should be eovered"); + return null; + } + + // TODO(datafusion) lots of hack here, should build our own Randomly later + public Node getRandomConstant(DataFusionGlobalState state) { + if (Randomly.getBooleanWithSmallProbability()) { + return DataFusionConstant.createNullConstant(); + } + switch (this) { + case BIGINT: + return DataFusionConstant.createIntConstant(state.getRandomly().getInteger()); + case BOOLEAN: + return new DataFusionConstant.DataFusionBooleanConstant(Randomly.getBoolean()); + case DOUBLE: + if (Randomly.getBoolean()) { + if (Randomly.getBoolean()) { + Double randomDouble = state.getRandomly().getDouble(); // [0.0, 1.0); + Double scaledDouble = (randomDouble - 0.5) * 2 * Double.MAX_VALUE; + return new DataFusionConstant.DataFusionDoubleConstant(scaledDouble); + } + String doubleStr = Randomly.fromOptions("'NaN'::Double", "'+Inf'::Double", "'-Inf'::Double", "-0.0", + "+0.0"); + return new DataFusionConstant.DataFusionDoubleConstant(doubleStr); + } + + return new DataFusionConstant.DataFusionDoubleConstant(state.getRandomly().getDouble()); + case NULL: + return DataFusionConstant.createNullConstant(); + default: + dfAssert(false, "Unreachable. All branches should be eovered"); + } + + dfAssert(false, "Unreachable. All branches should be eovered"); + return DataFusionConstant.createNullConstant(); + } + } + + public static class DataFusionColumn extends AbstractTableColumn { + + private final boolean isNullable; + + public DataFusionColumn(String name, DataFusionDataType columnType, boolean isNullable) { + super(name, null, columnType); + this.isNullable = isNullable; + } + + public boolean isNullable() { + return isNullable; + } + + } + + public static class DataFusionTable + extends AbstractRelationalTable { + + public DataFusionTable(String tableName, List columns, boolean isView) { + super(tableName, columns, Collections.emptyList(), isView); + } + + public static List getAllColumns(List tables) { + return tables.stream().map(AbstractTable::getColumns).flatMap(List::stream).collect(Collectors.toList()); + } + + public static List getRandomColumns(List tables) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new DataFusionColumn("*", DataFusionDataType.NULL, true)); + } + + List allColumns = getAllColumns(tables); + + return Randomly.nonEmptySubset(allColumns); + } + } + +} diff --git a/src/sqlancer/datafusion/DataFusionToStringVisitor.java b/src/sqlancer/datafusion/DataFusionToStringVisitor.java new file mode 100644 index 000000000..1f3030552 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionToStringVisitor.java @@ -0,0 +1,99 @@ +package sqlancer.datafusion; + +import java.util.List; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.Node; +import sqlancer.datafusion.ast.DataFusionConstant; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionJoin; +import sqlancer.datafusion.ast.DataFusionSelect; + +public class DataFusionToStringVisitor extends NewToStringVisitor { + + public static String asString(Node expr) { + DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } + + public static String asString(List> exprs) { + DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); + visitor.visit(exprs); + return visitor.get(); + } + + @Override + public void visitSpecific(Node expr) { + if (expr instanceof DataFusionConstant) { + visit((DataFusionConstant) expr); + } else if (expr instanceof DataFusionSelect) { + visit((DataFusionSelect) expr); + } else if (expr instanceof DataFusionJoin) { + visit((DataFusionJoin) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(DataFusionJoin join) { + visit(join.getLeftTable()); + sb.append(" "); + sb.append(join.getJoinType()); + sb.append(" "); + + sb.append(" JOIN "); + visit(join.getRightTable()); + if (join.getOnCondition() != null) { + sb.append(" ON "); + visit(join.getOnCondition()); + } + } + + private void visit(DataFusionConstant constant) { + sb.append(constant.toString()); + } + + private void visit(DataFusionSelect select) { + sb.append("SELECT "); + if (select.fetchColumnsString.isPresent()) { + sb.append(select.fetchColumnsString.get()); + } else { + visit(select.getFetchColumns()); + } + + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + +} diff --git a/src/sqlancer/datafusion/DataFusionUtil.java b/src/sqlancer/datafusion/DataFusionUtil.java new file mode 100644 index 000000000..e37ad8d64 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionUtil.java @@ -0,0 +1,193 @@ +package sqlancer.datafusion; + +import static java.lang.System.exit; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; + +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; + +public final class DataFusionUtil { + private DataFusionUtil() { + dfAssert(false, "Utility class cannot be instantiated"); + } + + // Display tables in `fromTableNames` + public static String displayTables(DataFusionGlobalState state, List fromTableNames) { + StringBuilder resultStringBuilder = new StringBuilder(); + for (String tableName : fromTableNames) { + String query = String.format("select * from %s", tableName); + try (Statement stat = state.getConnection().createStatement(); + ResultSet wholeTable = stat.executeQuery(query)) { + + ResultSetMetaData metaData = wholeTable.getMetaData(); + int columnCount = metaData.getColumnCount(); + + resultStringBuilder.append("Table: ").append(tableName).append("\n"); + for (int i = 1; i <= columnCount; i++) { + resultStringBuilder.append(metaData.getColumnName(i)).append(" (") + .append(metaData.getColumnTypeName(i)).append(")"); + if (i < columnCount) { + resultStringBuilder.append(", "); + } + } + resultStringBuilder.append("\n"); + + while (wholeTable.next()) { + for (int i = 1; i <= columnCount; i++) { + resultStringBuilder.append(wholeTable.getString(i)); + if (i < columnCount) { + resultStringBuilder.append(", "); + } + } + resultStringBuilder.append("\n"); + } + resultStringBuilder.append("----------------------------------------\n\n"); + + } catch (SQLException err) { + resultStringBuilder.append("Table: ").append(tableName).append("\n"); + resultStringBuilder.append("----------------------------------------\n\n"); + // resultStringBuilder.append("Error retrieving data from table ").append(tableName).append(": + // ").append(err.getMessage()).append("\n"); + } + } + + return resultStringBuilder.toString(); + } + + public static void dfAssert(boolean condition, String message) { + if (!condition) { + String methodName = Thread.currentThread().getStackTrace()[2].getMethodName(); + System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message); + exit(1); + } + } + + /* + * Fetch all DMLs from logs/database*-cur.log + */ + public static String getReplay(String dbname) { + String path = "./logs/datafusion/" + dbname + "-cur.log"; + String absolutePath = Paths.get(path).toAbsolutePath().toString(); + + StringBuilder reproducer = new StringBuilder(); + + try (BufferedReader reader = new BufferedReader(new FileReader(absolutePath))) { + String line; + while ((line = reader.readLine()) != null) { + // Check if the line contains the /*DML*/ marker + if (line.contains("/*DML*/")) { + reproducer.append(line).append("\n"); + } + } + } catch (IOException e) { + System.err.println("Error reading from file: " + e.getMessage()); + } + + return reproducer.toString(); + } + + // UID for different fuzzer runs + public static class DataFusionInstanceID { + private final String id; + + public DataFusionInstanceID(String dfID) { + id = dfID; + } + + @Override + public String toString() { + return id; // Return the id field when toString is called + } + } + + /* + * Extra logs stored in 'logs/datafusion_custom_log/' In case re-run overwrite previous logs + */ + public static class DataFusionLogger { + private final DataFusionInstanceID dfID; + private final DataFusionGlobalState state; + /* + * Log file handles + */ + private final File errorLogFile; + + public DataFusionLogger(DataFusionGlobalState globalState, DataFusionInstanceID id) throws Exception { + this.state = globalState; + this.dfID = id; + + // Setup datafusion_custom_log folder + File baseDir = new File("logs/datafusion_custom_log/"); + if (!baseDir.exists() && !baseDir.mkdirs()) { + throw new IOException("Failed to create 'datafusion_custom_log' directory/"); + } + + // Setup error.log + errorLogFile = new File(baseDir, "error_report.log"); + errorLogFile.createNewFile(); + } + + // Caller is responsible for adding '\n' at the end of logContent + public void appendToLog(DataFusionLogType logType, String logContent) { + FileWriter logFileWriter = null; + + // Determine which log file to use based on the LogType + String logLineHeader = ""; + switch (logType) { + case ERROR: + try { + logFileWriter = new FileWriter(errorLogFile, true); + } catch (IOException e) { + System.out.println("Failed to create FileWriter for errorLogFIle"); + e.printStackTrace(); + exit(1); + } + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + String formattedDateTime = LocalDateTime.now().format(formatter); + logLineHeader = "Run@" + formattedDateTime + " (" + dfID + ")\n"; + break; + case DML: + logFileWriter = state.getLogger().getCurrentFileWriter(); + logLineHeader = "/*DML*/"; + break; + case SELECT: + logFileWriter = state.getLogger().getCurrentFileWriter(); + break; + default: + dfAssert(false, "All branch should be covered"); + } + + // Append content to the appropriate log file + if (logFileWriter != null) { + try { + logFileWriter.write(logLineHeader); + logFileWriter.write(logContent); + logFileWriter.flush(); + } catch (IOException e) { + System.out.println("Failed to write to " + logType + " log: " + e.getMessage()); + e.printStackTrace(); + exit(1); + } + } else { + System.out.println("appending to log failed"); + Thread.currentThread().getStackTrace(); + exit(1); + } + } + + public enum DataFusionLogType { + ERROR, DML, SELECT + } + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionConstant.java b/src/sqlancer/datafusion/ast/DataFusionConstant.java new file mode 100644 index 000000000..d123adb8a --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionConstant.java @@ -0,0 +1,99 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.newast.Node; + +public class DataFusionConstant implements Node { + + private DataFusionConstant() { + } + + public static Node createIntConstant(long val) { + return new DataFusionIntConstant(val); + } + + public static Node createNullConstant() { + return new DataFusionNullConstant(); + } + + public static class DataFusionNullConstant extends DataFusionConstant { + + @Override + public String toString() { + return "NULL"; + } + + } + + public static class DataFusionIntConstant extends DataFusionConstant { + + private final long value; + + public DataFusionIntConstant(long value) { + this.value = value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + + public long getValue() { + return value; + } + + } + + public static class DataFusionDoubleConstant extends DataFusionConstant { + + private final String valueStr; + + public DataFusionDoubleConstant(double value) { + if (value == Double.POSITIVE_INFINITY) { + valueStr = "'+Inf'::Double"; + } else if (value == Double.NEGATIVE_INFINITY) { + valueStr = "'-Inf'::Double"; + } else if (Double.isNaN(value)) { + valueStr = "'NaN'::Double"; + } else if (Double.compare(value, -0.0) == 0) { + valueStr = "-0.0"; + } else { + valueStr = String.valueOf(value); + } + } + + // Make it more convenient to construct special value like -0, NaN, etc. + public DataFusionDoubleConstant(String valueStr) { + this.valueStr = valueStr; + } + + @Override + public String toString() { + return valueStr; + } + + } + + public static class DataFusionBooleanConstant extends DataFusionConstant { + + private final boolean value; + + public DataFusionBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + if (value) { + return "true"; + } else { + return "false"; + } + } + + } + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionExpression.java b/src/sqlancer/datafusion/ast/DataFusionExpression.java new file mode 100644 index 000000000..6da95cd93 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionExpression.java @@ -0,0 +1,5 @@ +package sqlancer.datafusion.ast; + +public interface DataFusionExpression { + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionJoin.java b/src/sqlancer/datafusion/ast/DataFusionJoin.java new file mode 100644 index 000000000..61cd9f556 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionJoin.java @@ -0,0 +1,92 @@ +package sqlancer.datafusion.ast; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +/* + NOT IMPLEMENTED YET + */ +public class DataFusionJoin implements Node { + + private final TableReferenceNode leftTable; + private final TableReferenceNode rightTable; + private final JoinType joinType; + private final Node onCondition; + + public DataFusionJoin(TableReferenceNode leftTable, + TableReferenceNode rightTable, JoinType joinType, + Node whereCondition) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onCondition = whereCondition; + } + + public static List> getJoins(List tables, + DataFusionGlobalState globalState) { + // [t1_join_t2, t1_join_t3, ...] + List> tableList = tables.stream() + .map(t -> new TableReferenceNode(t)) + .collect(Collectors.toList()); + List> joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + TableReferenceNode leftTable = tableList.remove(0); + TableReferenceNode rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + // TODO(datafusion) this `joinGen` can generate super chaotic exprsions, maybe we should make it more like a + // normal join expression + DataFusionExpressionGenerator joinGen = new DataFusionExpressionGenerator(globalState).setColumns(columns); + switch (DataFusionJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(DataFusionJoin.createInnerJoin(leftTable, rightTable, + joinGen.generateExpression(DataFusionSchema.DataFusionDataType.BOOLEAN))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public static DataFusionJoin createInnerJoin(TableReferenceNode left, + TableReferenceNode right, Node predicate) { + return new DataFusionJoin(left, right, JoinType.INNER, predicate); + } + + public TableReferenceNode getLeftTable() { + return leftTable; + } + + public TableReferenceNode getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public Node getOnCondition() { + return onCondition; + } + + public enum JoinType { + INNER; + // NATURAL, LEFT, RIGHT; + + public static JoinType getRandom() { + return Randomly.fromOptions(values()); + } + } + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionSelect.java b/src/sqlancer/datafusion/ast/DataFusionSelect.java new file mode 100644 index 000000000..a80758c4a --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionSelect.java @@ -0,0 +1,67 @@ +package sqlancer.datafusion.ast; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +public class DataFusionSelect extends SelectBase> implements Node { + public Optional fetchColumnsString = Optional.empty(); // When available, override `fetchColumns` in base + // class's `Node` representation (for display) + public DataFusionExpressionGenerator exprGen; + + // Construct a `DataFusionSelect` with random SELECT, FROM, WHERE + public static DataFusionSelect getRandomSelect(DataFusionGlobalState state) { + DataFusionSelect randomSelect = new DataFusionSelect(); + + // Randomly pick up to 4 tables to select from + DataFusionSchema schema = state.getSchema(); // schema of all tables + List allTables = schema.getDatabaseTables(); + List randomTables = Randomly.nonEmptySubset(allTables); + int maxSize = Randomly.fromOptions(1, 2, 3, 4); + if (randomTables.size() > maxSize) { + randomTables = randomTables.subList(0, maxSize); + } + + // Randomly choose some columns from `randomTables` + // And generate a random expression which might contain those columns + List randomColumns = DataFusionTable.getRandomColumns(randomTables); + randomSelect.exprGen = new DataFusionExpressionGenerator(state).setColumns(randomColumns); + Node whereExpr = randomSelect.exprGen + .generateExpression(DataFusionSchema.DataFusionDataType.BOOLEAN); + + // Constructing result + List> randomTableNodes = randomTables.stream() + .map(t -> new TableReferenceNode(t)) + .collect(Collectors.toList()); + List> randomColumnNodes = randomColumns.stream() + .map((c) -> new ColumnReferenceNode(c)) + .collect(Collectors.toList()); + + randomSelect.setFetchColumns(randomColumnNodes); + randomSelect.setFromList(randomTableNodes); + randomSelect.setWhereClause(whereExpr); + + return randomSelect; + } + + /* + * If set fetch columns with string It will override `fetchColumns` in base class when + * `DataFusionToStringVisitor.asString()` is called + * + * This method can be helpful to mutate select in oracle checks: SELECT [expr] ... -> SELECT SUM[expr] + */ + public void setFetchColumnsString(String selectExpr) { + this.fetchColumnsString = Optional.of(selectExpr); + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionBaseExpr.java b/src/sqlancer/datafusion/gen/DataFusionBaseExpr.java new file mode 100644 index 000000000..0be57486e --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionBaseExpr.java @@ -0,0 +1,258 @@ +package sqlancer.datafusion.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; + +/* + Notes for adding new `DataFusionBaseExpr` support: + + Expression ASTs are constructed with Node<> class, it can traverse expression and convert AST to String representation + `DataFusionBaseExpr` implements `Operator<>` class, which is a field inside `Node<>` class, it includes operator properties like number of arguments, signature, or is this operator prefix/suffix, etc. + + To add new base expr (scalar functions, operators like '<<', 'AND' are all base expr): + 1. Add an enum variant to `DataFusionBaseExprType` + 2. Update `DataFusionBaseExprFactory.java` + (If a function support different argument number, make a new entry for each one. e.g. round scalar function support round(3.14) / round(3.14, 1), so it should be enum FUNC_ROUND1, FUNC_ROUND2) + */ +public class DataFusionBaseExpr implements Operator { + public String name; + public int nArgs; // number of input arguments + public DataFusionBaseExprCategory exprType; + public List possibleReturnTypes; + public List argTypes; + public boolean isVariadic; // Function supports arbitrary number of arguments, if set to `true`, it will + // override `nArgs` + + // Primary constructor + DataFusionBaseExpr(String name, int nArgs, DataFusionBaseExprCategory exprCategory, + List possibleReturnTypes, List argTypes, boolean isVariadic) { + this.name = name; + this.nArgs = nArgs; + this.exprType = exprCategory; + this.possibleReturnTypes = possibleReturnTypes; + this.argTypes = argTypes; + this.isVariadic = isVariadic; + } + + // Overloaded constructor assuming 'isVariadic' is false + DataFusionBaseExpr(String name, int nArgs, DataFusionBaseExprCategory exprCategory, + List possibleReturnTypes, List argTypes) { + this(name, nArgs, exprCategory, possibleReturnTypes, argTypes, false); + } + + public static DataFusionBaseExpr createCommonNumericFuncSingleArg(String name) { + return new DataFusionBaseExpr(name, 1, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + } + + public static DataFusionBaseExpr createCommonNumericAggrFuncSingleArg(String name) { + return new DataFusionBaseExpr(name, 1, DataFusionBaseExprCategory.AGGREGATE, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + } + + public static DataFusionBaseExpr createCommonNumericFuncTwoArgs(String name) { + return new DataFusionBaseExpr(name, 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + } + + @Override + public String getTextRepresentation() { + return name; + } + + @Override + public String toString() { + return name; + } + + /* + * Class/Enum for `DataFusionBaseExpr` fields + */ + // Used to construct `src.common.ast.*Node` + public enum DataFusionBaseExprCategory { + UNARY_PREFIX, UNARY_POSTFIX, BINARY, FUNC, AGGREGATE + } + + /* + * Operators reference: https://datafusion.apache.org/user-guide/sql/operators.html Scalar functions: + * https://datafusion.apache.org/user-guide/sql/scalar_functions.html + */ + public enum DataFusionBaseExprType { + // Null Operators + IS_NULL, // IS NULL + IS_NOT_NULL, // IS NOT NULL + + // Numeric Operators + ADD, // 1 + 1 + SUB, // 1 - 1 + MULTIPLICATION, // 2 * 3 + DIVISION, // 8 / 4 + MODULO, // 5 % 3 + + // Comparison Operators + EQUAL, // 1 = 1 + EQUAL2, // 1 == 1 + NOT_EQUAL, // 1 != 2 + LESS_THAN, // 3 < 4 + LESS_THAN_OR_EQUAL_TO, // 3 <= 3 + GREATER_THAN, // 6 > 5 + GREATER_THAN_OR_EQUAL_TO, // 5 >= 5 + + // Distinctness operators + IS_DISTINCT_FROM, // 0 IS DISTINCT FROM NULL + IS_NOT_DISTINCT_FROM, // NULL IS NOT DISTINCT FROM NULL + + /* + * // Regular expression match operators REGEX_MATCH, // 'datafusion' ~ '^datafusion(-cli)*' + * REGEX_CASE_INSENSITIVE_MATCH, // 'datafusion' ~* '^DATAFUSION(-cli)*' NOT_REGEX_MATCH, // 'datafusion' !~ + * '^DATAFUSION(-cli)*' NOT_REGEX_CASE_INSENSITIVE_MATCH, // 'datafusion' !~* '^DATAFUSION(-cli)+' + * + * // Like pattern match operators LIKE_MATCH, // 'datafusion' ~~ 'dat_f%n' CASE_INSENSITIVE_LIKE_MATCH, // + * 'datafusion' ~~* 'Dat_F%n' NOT_LIKE_MATCH, // 'datafusion' !~~ 'Dat_F%n' NOT_CASE_INSENSITIVE_LIKE_MATCH // + * 'datafusion' !~~* 'Dat%F_n' + */ + + // Logical Operators + AND, // true and true + OR, // true or false + + // Bitwise Operators + BITWISE_AND, // 5 & 3 + BITWISE_OR, // 5 | 3 + BITWISE_XOR, // 5 ^ 3 + BITWISE_SHIFT_RIGHT, // 5 >> 3 + BITWISE_SHIFT_LEFT, // 5 << 3 + + /* + * // Other operators STRING_CONCATENATION, // 'Hello, ' || 'DataFusion!' ARRAY_CONTAINS, // + * make_array(1,2,3) @> make_array(1,3) ARRAY_IS_CONTAINED_BY // make_array(1,3) <@ make_array(1,2,3) + */ + + // Unary Prefix Operators + NOT, // NOT true + PLUS, // +7 + MINUS, // -3 + + /* + * Scalar Functions + */ + + // Math Functions + FUNC_ABS, // abs(-10) + FUNC_ACOS, // acos(1) + FUNC_ACOSH, // acosh(10) + FUNC_ASIN, // asin(1) + FUNC_ASINH, // asinh(1) + FUNC_ATAN, // atan(1) + FUNC_ATANH, // atanh(0.5) + FUNC_ATAN2, // atan2(10, 10) + FUNC_CBRT, // cbrt(27) + FUNC_CEIL, // ceil(9.2) + FUNC_COS, // cos(π/3) + FUNC_COSH, // cosh(0) + FUNC_DEGREES, // degrees(π) + FUNC_EXP, // exp(1) + FUNC_FACTORIAL, // factorial(5) + FUNC_FLOOR, // floor(3.7) + FUNC_GCD, // gcd(8, 12) + FUNC_ISNAN, // isnan(NaN) + FUNC_ISZERO, // iszero(0.0) + FUNC_LCM, // lcm(5, 15) + FUNC_LN, // ln(1) + FUNC_LOG, // log(100) + FUNC_LOG_WITH_BASE, // log(10, 100) + FUNC_LOG10, // log10(100) + FUNC_LOG2, // log2(32) + FUNC_NANVL, // nanvl(NaN, 3) + FUNC_PI, // pi() + FUNC_POW, // pow(2, 3) + FUNC_POWER, // power(2, 3) + FUNC_RADIANS, // radians(180) + // FUNC_RANDOM, // random() disabled because it's non-deterministic + FUNC_ROUND, // round(3.14159) + FUNC_ROUND_WITH_DECIMAL, // round(3.14159, 2) + FUNC_SIGNUM, // signum(-10) + FUNC_SIN, // sin(π/2) + FUNC_SINH, // sinh(1) + FUNC_SQRT, // sqrt(16) + FUNC_TAN, // tan(π/4) + FUNC_TANH, // tanh(1) + FUNC_TRUNC, // trunc(3.14159) + FUNC_TRUNC_WITH_DECIMAL, // trunc(3.14159, 2) + + // Conditional Functions + FUNC_COALESCE, // coalesce(NULL, 'default value') + FUNC_NULLIF, // nullif('value', 'value') + FUNC_NVL, // nvl(NULL, 'default value') + FUNC_NVL2, // nvl2('not null', 'return if not null', 'return if null') + FUNC_IFNULL, // ifnull(NULL, 'default value') + + // String Functions + + // Time and Date Functions + + // Array Functions + + // Struct Functions + + // Hashing Functions + + // Other Functions + + // Aggregate Functions + AGGR_MIN, AGGR_MAX, AGGR_SUM, AGGR_AVG, AGGR_COUNT, + } + + /* + * Because expressions are constructed in a top-down way, we have to infer argument type given return type. For each + * arg, if its corresponding element is `SameAsReturnType`, it should be the same as the type of expression's + * evaluated value. Else, it should be specific `DataFusionDataType` + * + * e.g. let's say we're generating a round(num, digit) of double type, its `argTypes` is: Arrays.asList( new + * ArgumentType.SameAsReturnType(), // First arg type as return type new ArgumentType.Fixed(new + * ArrayList<>(Array.asList(DataFusionDataType.INT)) // Second arg always Integer ) it means: its first argument + * should be the same as returned type (double), and the second arg should always be Int. + * + * Random expression generator's policy: SameAsReturnType -> generate an expr with the same type as its return type + * SameAsReturnType -> generate an expr with the same type as its 1st arg type Fixed(type1, type2, ... typeN) -> + * randomly choose a possible type (It will also generate completely random type/null ~10%) + * + * Note this defination is not comprehensive for native `DataFusion` types. It's just for simplicity and should + * cover most common cases + */ + public abstract static class ArgumentType { + private ArgumentType() { + } + + public static class SameAsReturnType extends ArgumentType { + } + + public static class SameAsFirstArgType extends ArgumentType { + } + + public static class Fixed extends ArgumentType { + public List fixedType; // It's a list to support different possible arg types. + + public Fixed(List fixedType) { + this.fixedType = fixedType; + } + + public List getType() { + return fixedType; + } + } + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java b/src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java new file mode 100644 index 000000000..d3fe39972 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionBaseExprFactory.java @@ -0,0 +1,391 @@ +package sqlancer.datafusion.gen; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; +import static sqlancer.datafusion.gen.DataFusionBaseExpr.createCommonNumericAggrFuncSingleArg; +import static sqlancer.datafusion.gen.DataFusionBaseExpr.createCommonNumericFuncSingleArg; +import static sqlancer.datafusion.gen.DataFusionBaseExpr.createCommonNumericFuncTwoArgs; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; +import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprCategory; +import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; + +public final class DataFusionBaseExprFactory { + private DataFusionBaseExprFactory() { + dfAssert(false, "Utility class cannot be instantiated"); + } + + public static DataFusionBaseExpr createExpr(DataFusionBaseExprType type) { + switch (type) { + case IS_NULL: + return new DataFusionBaseExpr("IS NULL", 1, DataFusionBaseExprCategory.UNARY_POSTFIX, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.DOUBLE, DataFusionDataType.BIGINT, DataFusionDataType.NULL))))); + case IS_NOT_NULL: + return new DataFusionBaseExpr("IS NOT NULL", 1, DataFusionBaseExprCategory.UNARY_POSTFIX, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.DOUBLE, DataFusionDataType.BIGINT, DataFusionDataType.NULL))))); + case BITWISE_AND: + return new DataFusionBaseExpr("&", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case BITWISE_OR: + return new DataFusionBaseExpr("|", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case BITWISE_XOR: + return new DataFusionBaseExpr("^", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case BITWISE_SHIFT_RIGHT: + return new DataFusionBaseExpr(">>", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case BITWISE_SHIFT_LEFT: + return new DataFusionBaseExpr("<<", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case NOT: + return new DataFusionBaseExpr("NOT", 1, DataFusionBaseExprCategory.UNARY_PREFIX, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))))); + case PLUS: // unary prefix '+' + return new DataFusionBaseExpr("+", 1, DataFusionBaseExprCategory.UNARY_PREFIX, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case MINUS: // unary prefix '-' + return new DataFusionBaseExpr("-", 1, DataFusionBaseExprCategory.UNARY_PREFIX, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed( + new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case MULTIPLICATION: + return new DataFusionBaseExpr("*", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case DIVISION: + return new DataFusionBaseExpr("/", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case MODULO: + return new DataFusionBaseExpr("%", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case EQUAL: + return new DataFusionBaseExpr("=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case EQUAL2: + return new DataFusionBaseExpr("==", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case NOT_EQUAL: + return new DataFusionBaseExpr("!=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case LESS_THAN: + return new DataFusionBaseExpr("<", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case LESS_THAN_OR_EQUAL_TO: + return new DataFusionBaseExpr("<=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case GREATER_THAN: + return new DataFusionBaseExpr(">", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case GREATER_THAN_OR_EQUAL_TO: + return new DataFusionBaseExpr(">=", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case IS_DISTINCT_FROM: + return new DataFusionBaseExpr("IS DISTINCT FROM", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case IS_NOT_DISTINCT_FROM: + return new DataFusionBaseExpr("IS NOT DISTINCT FROM", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT, + DataFusionDataType.DOUBLE, DataFusionDataType.BOOLEAN))), + new ArgumentType.SameAsFirstArgType())); + case AND: + return new DataFusionBaseExpr("AND", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))), // arg1 + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))) // arg2 + )); + case OR: + return new DataFusionBaseExpr("OR", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BOOLEAN), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))), // arg1 + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN))) // arg2 + )); + case ADD: // binary arithmetic operator '+' + return new DataFusionBaseExpr("+", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))), // arg1 + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))) // arg2 + )); + case SUB: // binary arithmetic operator '-' + return new DataFusionBaseExpr("-", 2, DataFusionBaseExprCategory.BINARY, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), // arg1 + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))) // arg2 + )); + case FUNC_ABS: + return createCommonNumericFuncSingleArg("ABS"); + case FUNC_ACOS: + return createCommonNumericFuncSingleArg("ACOS"); + case FUNC_ACOSH: + return createCommonNumericFuncSingleArg("ACOSH"); + case FUNC_ASIN: + return createCommonNumericFuncSingleArg("ASIN"); + case FUNC_ASINH: + return createCommonNumericFuncSingleArg("ASINH"); + case FUNC_ATAN: + return createCommonNumericFuncSingleArg("ATAN"); + case FUNC_ATANH: + return createCommonNumericFuncSingleArg("ATANH"); + case FUNC_ATAN2: + return createCommonNumericFuncTwoArgs("ATAN2"); + case FUNC_CBRT: + return createCommonNumericFuncSingleArg("CBRT"); + case FUNC_CEIL: + return createCommonNumericFuncSingleArg("CEIL"); + case FUNC_COS: + return createCommonNumericFuncSingleArg("COS"); + case FUNC_COSH: + return createCommonNumericFuncSingleArg("COSH"); + case FUNC_DEGREES: + return createCommonNumericFuncSingleArg("DEGREES"); + case FUNC_EXP: + return createCommonNumericFuncSingleArg("EXP"); + case FUNC_FACTORIAL: + return createCommonNumericFuncSingleArg("FACTORIAL"); + case FUNC_FLOOR: + return createCommonNumericFuncSingleArg("FLOOR"); + case FUNC_GCD: + return new DataFusionBaseExpr("GCD", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_ISNAN: + return createCommonNumericFuncSingleArg("ISNAN"); + case FUNC_ISZERO: + return createCommonNumericFuncSingleArg("ISZERO"); + case FUNC_LCM: + return createCommonNumericFuncTwoArgs("LCM"); + case FUNC_LN: + return createCommonNumericFuncSingleArg("LN"); + case FUNC_LOG: + return createCommonNumericFuncSingleArg("LOG"); + case FUNC_LOG_WITH_BASE: + return createCommonNumericFuncTwoArgs("LOG"); + case FUNC_LOG10: + return createCommonNumericFuncSingleArg("LOG10"); + case FUNC_LOG2: + return createCommonNumericFuncSingleArg("LOG2"); + case FUNC_NANVL: + return createCommonNumericFuncTwoArgs("NANVL"); + case FUNC_PI: + return new DataFusionBaseExpr("PI", 0, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), Arrays.asList()); + case FUNC_POW: + return createCommonNumericFuncSingleArg("POW"); + case FUNC_POWER: + return createCommonNumericFuncSingleArg("POWER"); + case FUNC_RADIANS: + return createCommonNumericFuncSingleArg("RADIANS"); + case FUNC_ROUND: + return createCommonNumericFuncSingleArg("ROUND"); + case FUNC_ROUND_WITH_DECIMAL: + return new DataFusionBaseExpr("ROUND", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case FUNC_SIGNUM: + return createCommonNumericFuncSingleArg("SIGNUM"); + case FUNC_SIN: + return createCommonNumericFuncSingleArg("SIN"); + case FUNC_SINH: + return createCommonNumericFuncSingleArg("SINH"); + case FUNC_SQRT: + return createCommonNumericFuncSingleArg("SQRT"); + case FUNC_TAN: + return createCommonNumericFuncSingleArg("TAN"); + case FUNC_TANH: + return createCommonNumericFuncSingleArg("TANH"); + case FUNC_TRUNC: + return createCommonNumericFuncSingleArg("TRUNC"); + case FUNC_TRUNC_WITH_DECIMAL: + return new DataFusionBaseExpr("TRUNC", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>( + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BIGINT))))); + case FUNC_COALESCE: + return new DataFusionBaseExpr("COALESCE", -1, // overide by variadic + DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), Arrays.asList(), true); + case FUNC_NULLIF: + return new DataFusionBaseExpr("NULLIF", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_NVL: + return new DataFusionBaseExpr("NVL", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_NVL2: + return new DataFusionBaseExpr("NVL2", 3, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + case FUNC_IFNULL: + return new DataFusionBaseExpr("IFNULL", 2, DataFusionBaseExprCategory.FUNC, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList( + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))), + new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE))))); + + case AGGR_MIN: + return createCommonNumericAggrFuncSingleArg("MIN"); + case AGGR_MAX: + return createCommonNumericAggrFuncSingleArg("MAX"); + case AGGR_AVG: + return createCommonNumericAggrFuncSingleArg("AVG"); + case AGGR_SUM: + return createCommonNumericAggrFuncSingleArg("SUM"); + case AGGR_COUNT: + return new DataFusionBaseExpr("COUNT", -1, DataFusionBaseExprCategory.AGGREGATE, + Arrays.asList(DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE), + Arrays.asList(new ArgumentType.Fixed(new ArrayList<>(Arrays.asList(DataFusionDataType.BOOLEAN, + DataFusionDataType.BIGINT, DataFusionDataType.DOUBLE)))), + true); + default: + dfAssert(false, "Unreachable. Unimplemented branch for type " + type); + } + + dfAssert(false, "Unreachable. Unimplemented branch for type " + type); + return null; + } + + // if input is Optional.empty(), return all possible `DataFusionBaseExpr`s + // else, return all `DataFusionBaseExpr` which might be evaluated to arg's type + public static List getExprsWithReturnType(Optional dataTypeOptional) { + List allExpressions = Arrays.stream(DataFusionBaseExprType.values()) + .map(DataFusionBaseExprFactory::createExpr).collect(Collectors.toList()); + + if (!dataTypeOptional.isPresent()) { + return allExpressions; // If Optional is empty, return all expressions + } + + DataFusionDataType filterType = dataTypeOptional.get(); + List exprsWithReturnType = allExpressions.stream() + .filter(expr -> expr.possibleReturnTypes.contains(filterType)).collect(Collectors.toList()); + + if (Randomly.getBoolean()) { + // Too many similar function, so test them less often + return exprsWithReturnType; + } + + return exprsWithReturnType.stream().filter(expr -> expr.exprType != DataFusionBaseExprCategory.FUNC) + .collect(Collectors.toList()); + } + + public static DataFusionBaseExpr getRandomAggregateExpr() { + List allAggrExpressions = Arrays.stream(DataFusionBaseExprType.values()) + .map(DataFusionBaseExprFactory::createExpr) + .filter(expr -> expr.exprType == DataFusionBaseExprCategory.AGGREGATE).collect(Collectors.toList()); + + return Randomly.fromList(allAggrExpressions); + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java new file mode 100644 index 000000000..3d7340615 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java @@ -0,0 +1,227 @@ +package sqlancer.datafusion.gen; + +import static sqlancer.datafusion.DataFusionUtil.dfAssert; +import static sqlancer.datafusion.gen.DataFusionBaseExprFactory.createExpr; +import static sqlancer.datafusion.gen.DataFusionBaseExprFactory.getExprsWithReturnType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; +import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; + +public final class DataFusionExpressionGenerator + extends TypedExpressionGenerator, DataFusionColumn, DataFusionDataType> { + + private final DataFusionGlobalState globalState; + + public DataFusionExpressionGenerator(DataFusionGlobalState globalState) { + this.globalState = globalState; + } + + @Override + protected DataFusionDataType getRandomType() { + DataFusionDataType dt; + do { + dt = Randomly.fromOptions(DataFusionDataType.values()); + } while (dt == DataFusionDataType.NULL); + + return dt; + } + + @Override + protected boolean canGenerateColumnOfType(DataFusionDataType type) { + return true; + } + + @Override + protected Node generateExpression(DataFusionDataType type, int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + DataFusionDataType expectedType = type; + if (Randomly.getBooleanWithRatherLowProbability()) { // ~10% + expectedType = DataFusionDataType.getRandomWithoutNull(); + } + return generateLeafNode(expectedType); + } + + // nested aggregate is not allowed, so occasionally apply it + Boolean includeAggr = Randomly.getBooleanWithSmallProbability(); + List possibleBaseExprs = getExprsWithReturnType(Optional.of(type)).stream() + // Conditinally apply filter if `includeAggr` set to false + .filter(expr -> includeAggr || expr.exprType != DataFusionBaseExpr.DataFusionBaseExprCategory.AGGREGATE) + .collect(Collectors.toList()); + + if (possibleBaseExprs.isEmpty()) { + dfAssert(type == DataFusionDataType.NULL, "should able to generate expression with type " + type); + return generateLeafNode(type); + } + + DataFusionBaseExpr randomExpr = Randomly.fromList(possibleBaseExprs); + switch (randomExpr.exprType) { + case UNARY_PREFIX: + DataFusionDataType argType = null; + dfAssert(randomExpr.argTypes.size() == 1 && randomExpr.nArgs == 1, + "Unary expression should only have 1 argument" + randomExpr.argTypes); + if (randomExpr.argTypes.get(0) instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); + argType = Randomly.fromList(possibleArgTypes.fixedType); + } else { + argType = type; + } + + return new NewUnaryPrefixOperatorNode(generateExpression(argType, depth + 1), + randomExpr); + case UNARY_POSTFIX: + dfAssert(randomExpr.argTypes.size() == 1 && randomExpr.nArgs == 1, + "Unary expression should only have 1 argument" + randomExpr.argTypes); + if (randomExpr.argTypes.get(0) instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); + argType = Randomly.fromList(possibleArgTypes.fixedType); + } else { + argType = type; + } + + return new NewUnaryPostfixOperatorNode(generateExpression(argType, depth + 1), + randomExpr); + case BINARY: + dfAssert(randomExpr.argTypes.size() == 2 && randomExpr.nArgs == 2, + "Binrary expression should only have 2 argument" + randomExpr.argTypes); + List argTypeList = new ArrayList<>(); // types of current expression's input arguments + for (ArgumentType argumentType : randomExpr.argTypes) { + if (argumentType instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); + dfAssert(!possibleArgTypes.fixedType.isEmpty(), "possible types can't be an empty list"); + DataFusionDataType determinedType = Randomly.fromList(possibleArgTypes.fixedType); + argTypeList.add(determinedType); + } else if (argumentType instanceof ArgumentType.SameAsFirstArgType) { + dfAssert(!argTypeList.isEmpty(), "First argument can't have argument type `SameAsFirstArgType`"); + DataFusionDataType firstArgType = argTypeList.get(0); + argTypeList.add(firstArgType); + } else { + // Same as expression return type + argTypeList.add(type); + } + } + + return new NewBinaryOperatorNode(generateExpression(argTypeList.get(0), depth + 1), + generateExpression(argTypeList.get(1), depth + 1), randomExpr); + case AGGREGATE: + // Fall through + case FUNC: + return generateFunctionExpression(type, depth, randomExpr); + default: + dfAssert(false, "unreachable"); + } + + dfAssert(false, "unreachable"); + return null; + } + + public Node generateFunctionExpression(DataFusionDataType type, int depth, + DataFusionBaseExpr exprType) { + if (exprType.isVariadic || Randomly.getBooleanWithSmallProbability()) { + // TODO(datafusion) maybe add possible types. e.g. some function have signature variadic(INT/DOUBLE), then + // only randomly pick from INT and DOUBLE + int nArgs = Randomly.smallNumber(); // 0, 2, 4, ... smaller one is more likely + return new NewFunctionNode(generateExpressions(nArgs), exprType); + } + + List funcArgTypeList = new ArrayList<>(); // types of current expression's input arguments + int i = 0; + for (ArgumentType argumentType : exprType.argTypes) { + if (argumentType instanceof ArgumentType.Fixed) { + ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) exprType.argTypes.get(i); + dfAssert(!possibleArgTypes.fixedType.isEmpty(), "possible types can't be an empty list"); + DataFusionDataType determinedType = Randomly.fromList(possibleArgTypes.fixedType); + funcArgTypeList.add(determinedType); + } else if (argumentType instanceof ArgumentType.SameAsFirstArgType) { + dfAssert(!funcArgTypeList.isEmpty(), "First argument can't have argument type `SameAsFirstArgType`"); + DataFusionDataType firstArgType = funcArgTypeList.get(0); + funcArgTypeList.add(firstArgType); + } else { + // Same as expression return type + funcArgTypeList.add(type); + } + i++; + } + + List> argExpressions = new ArrayList<>(); + + for (DataFusionDataType dataType : funcArgTypeList) { + argExpressions.add(generateExpression(dataType, depth + 1)); + } + + return new NewFunctionNode(argExpressions, exprType); + } + + List filterColumns(DataFusionDataType type) { + if (columns == null) { + return Collections.emptyList(); + } else { + return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); + } + } + + @Override + protected Node generateColumn(DataFusionDataType type) { + // HACK: if no col of such type exist, generate constant value instead + List colsOfType = filterColumns(type); + if (colsOfType.isEmpty()) { + return generateConstant(type); + } + + DataFusionColumn column = Randomly.fromList(colsOfType); + return new ColumnReferenceNode(column); + } + + @Override + public Node generateConstant(DataFusionDataType type) { + return type.getRandomConstant(globalState); + } + + @Override + public Node generatePredicate() { + return generateExpression(DataFusionDataType.BOOLEAN, 0); + } + + @Override + public Node negatePredicate(Node predicate) { + return new NewUnaryPrefixOperatorNode<>(predicate, createExpr(DataFusionBaseExprType.NOT)); + } + + @Override + public Node isNull(Node expr) { + return new NewUnaryPostfixOperatorNode<>(expr, createExpr(DataFusionBaseExprType.IS_NULL)); + } + + public static class DataFusionCastOperation extends NewUnaryPostfixOperatorNode { + + public DataFusionCastOperation(Node expr, DataFusionDataType type) { + super(expr, new Operator() { + + @Override + public String getTextRepresentation() { + return "::" + type.toString(); + } + }); + } + + } + +} diff --git a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java new file mode 100644 index 000000000..1ee00dd50 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java @@ -0,0 +1,54 @@ +package sqlancer.datafusion.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionToStringVisitor; + +public class DataFusionInsertGenerator extends AbstractInsertGenerator { + + private final DataFusionGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + + public DataFusionInsertGenerator(DataFusionGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(DataFusionGlobalState globalState, DataFusionTable targetTable) { + return new DataFusionInsertGenerator(globalState).generate(targetTable); + } + + private SQLQueryAdapter generate(DataFusionTable targetTable) { + // `sb` is a global `StringBuilder` for current insert query + sb.append("INSERT INTO "); + + if (targetTable.getColumns().isEmpty()) { + throw new IgnoreMeException(); + } + List columns = targetTable.getRandomNonEmptyColumnSubset(); + + sb.append(targetTable.getName()); + sb.append("("); + sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); + sb.append(")"); + sb.append(" VALUES "); + insertColumns(columns); // will finally call `insertValue()` to generate random value + + return new SQLQueryAdapter(sb.toString(), errors); + } + + @Override + protected void insertValue(DataFusionColumn col) { + String val = DataFusionToStringVisitor + .asString(new DataFusionExpressionGenerator(globalState).generateConstant(col.getType())); + sb.append(val); + } + +} diff --git a/src/sqlancer/datafusion/gen/DataFusionTableGenerator.java b/src/sqlancer/datafusion/gen/DataFusionTableGenerator.java new file mode 100644 index 000000000..adececaa7 --- /dev/null +++ b/src/sqlancer/datafusion/gen/DataFusionTableGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.datafusion.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; + +public class DataFusionTableGenerator { + + // Randomly generate a query like 'create table t1 (v1 bigint, v2 boolean)' + public SQLQueryAdapter getQuery(DataFusionGlobalState globalState) { + ExpectedErrors errors = new ExpectedErrors(); + StringBuilder sb = new StringBuilder(); + String tableName = globalState.getSchema().getFreeTableName(); + sb.append("CREATE TABLE "); + sb.append(tableName); + sb.append("("); + + int colCount = Randomly.smallNumber() + 1 + (Randomly.getBoolean() ? 1 : 0); + for (int i = 0; i < colCount; i++) { + sb.append("v").append(i).append(" ").append(DataFusionDataType.getRandomWithoutNull().toString()); + + if (i != colCount - 1) { + sb.append(", "); + } + } + + sb.append(");"); + + return new SQLQueryAdapter(sb.toString(), errors, true); + } +} diff --git a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml new file mode 100644 index 000000000..ebff15b35 --- /dev/null +++ b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "datafusion-server" +edition = "2021" +description = "Standalone DataFusion server" +license = "Apache-2.0" + +[dependencies] +ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] } +arrow = { version = "52.1.0", features = ["prettyprint"] } +arrow-array = { version = "52.1.0", default-features = false, features = ["chrono-tz"] } +arrow-buffer = { version = "52.1.0", default-features = false } +arrow-flight = { version = "52.1.0", features = ["flight-sql-experimental"] } +arrow-ipc = { version = "52.1.0", default-features = false, features = ["lz4"] } +arrow-ord = { version = "52.1.0", default-features = false } +arrow-schema = { version = "52.1.0", default-features = false } +arrow-string = { version = "52.1.0", default-features = false } +async-trait = "0.1.73" +bytes = "1.4" +chrono = { version = "0.4.34", default-features = false } +dashmap = "5.5.0" +datafusion_stable = { package = "datafusion", git = "https://github.com/apache/datafusion.git", rev = "e693ed7", optional = true } +datafusion_dev = { package = "datafusion", git = "https://github.com/apache/datafusion.git", branch = "main", optional = true } +env_logger = "0.11" +futures = "0.3" +half = { version = "2.2.1", default-features = false } +hashbrown = { version = "0.14.5", features = ["raw"] } +log = "0.4" +num_cpus = "1.13.0" +object_store = { version = "0.10.1", default-features = false } +parking_lot = "0.12" +parquet = { version = "52.0.0", default-features = false, features = ["arrow", "async", "object_store"] } +rand = "0.8" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1" +tokio = { version = "1.36", features = ["macros", "rt", "sync"] } +tonic = "0.11" +uuid = "1.0" +prost = { version = "0.12", default-features = false } +prost-derive = { version = "0.12", default-features = false } +mimalloc = { version = "0.1", default-features = false } + +[[bin]] +name = "datafusion-server" +path = "src/main.rs" + +[features] +default = ["datafusion_stable_feature"] +datafusion_stable_feature = ["datafusion_stable"] +datafusion_dev_feature = ["datafusion_dev"] diff --git a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs new file mode 100644 index 000000000..c19728bb6 --- /dev/null +++ b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs @@ -0,0 +1,463 @@ +#[cfg(feature = "datafusion_dev")] +extern crate datafusion_dev as datafusion; +#[cfg(feature = "datafusion_stable")] +extern crate datafusion_stable as datafusion; + +use arrow::array::{ArrayRef, StringArray}; +use arrow::ipc::writer::IpcWriteOptions; +use arrow::record_batch::RecordBatch; +use arrow_flight::encode::FlightDataEncoderBuilder; +use arrow_flight::flight_descriptor::DescriptorType; +use arrow_flight::flight_service_server::{FlightService, FlightServiceServer}; +use arrow_flight::sql::server::{FlightSqlService, PeekableFlightDataStream}; +use arrow_flight::sql::{ + ActionClosePreparedStatementRequest, ActionCreatePreparedStatementRequest, + ActionCreatePreparedStatementResult, Any, CommandGetTables, CommandPreparedStatementQuery, + CommandPreparedStatementUpdate, ProstMessageExt, SqlInfo, +}; +use arrow_flight::{ + Action, FlightDescriptor, FlightEndpoint, FlightInfo, HandshakeRequest, HandshakeResponse, + IpcMessage, SchemaAsIpc, Ticket, +}; +use arrow_schema::{DataType, Field, Schema}; +use dashmap::DashMap; +use datafusion_stable::logical_expr::LogicalPlan; +use datafusion_stable::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; +use futures::{Stream, StreamExt, TryStreamExt}; +use log::info; +use mimalloc::MiMalloc; +use prost::Message; +use std::pin::Pin; +use std::sync::Arc; +use tokio::sync::Mutex; +use tonic::metadata::MetadataValue; +use tonic::transport::Server; +use tonic::{Request, Response, Status, Streaming}; +use uuid::Uuid; + +#[global_allocator] +static GLOBAL: MiMalloc = MiMalloc; + +macro_rules! status { + ($desc:expr, $err:expr) => { + Status::internal(format!("{}: {} at {}:{}", $desc, $err, file!(), line!())) + }; +} + +/// Adapted from https://github.com/apache/datafusion/blob/main/datafusion-examples/examples/flight/flight_sql_server.rs +/// Can be used as a remote DataFusion server and connected by `JDBC` from client +/// Supported SQL statements: +/// CREATE +/// INSERT +/// SELECT +/// +/// Only single client is supported +/// For now use `ctx` instead of `contexts` inside `FlightSqlServiceImpl` +/// +/// === Below origianl comment === +/// +/// This example shows how to wrap DataFusion with `FlightSqlService` to support connecting +/// to a standalone DataFusion-based server with a JDBC client, using the open source "JDBC Driver +/// for Arrow Flight SQL". +/// +/// To install the JDBC driver in DBeaver for example, see these instructions: +/// https://docs.dremio.com/software/client-applications/dbeaver/ +/// When configuring the driver, specify property "UseEncryption" = false +/// +/// JDBC connection string: "jdbc:arrow-flight-sql://127.0.0.1:50051/" +/// +/// Based heavily on Ballista's implementation: https://github.com/apache/datafusion-ballista/blob/main/ballista/scheduler/src/flight_sql.rs +/// and the example in arrow-rs: https://github.com/apache/arrow-rs/blob/master/arrow-flight/examples/flight_sql_server.rs +/// +#[tokio::main] +async fn main() -> Result<(), Box> { + env_logger::init(); + let addr = "0.0.0.0:50051".parse()?; + let session_ctx = SessionContext::new_with_config( + SessionConfig::new().with_information_schema(true), // enable catalog + ); + let service = FlightSqlServiceImpl { + contexts: Default::default(), + statements: Default::default(), + results: Default::default(), + ctx: Arc::new(Mutex::new(session_ctx)), + }; + info!("Listening on {addr:?}"); + let svc = FlightServiceServer::new(service); + + Server::builder().add_service(svc).serve(addr).await?; + + Ok(()) +} + +pub struct FlightSqlServiceImpl { + contexts: Arc>>, + statements: Arc>, + results: Arc>>, + ctx: Arc>, +} + +impl FlightSqlServiceImpl { + async fn create_ctx(&self) -> Result { + let uuid = Uuid::new_v4().hyphenated().to_string(); + let session_config = SessionConfig::from_env() + .map_err(|e| Status::internal(format!("Error building plan: {e}")))? + .with_information_schema(true); + let ctx = Arc::new(SessionContext::new_with_config(session_config)); + + self.contexts.insert(uuid.clone(), ctx); + Ok(uuid) + } + + fn get_ctx(&self, req: &Request) -> Result, Status> { + // get the token from the authorization header on Request + let auth = req + .metadata() + .get("authorization") + .ok_or_else(|| Status::internal("No authorization header!"))?; + let str = auth + .to_str() + .map_err(|e| Status::internal(format!("Error parsing header: {e}")))?; + let authorization = str.to_string(); + let bearer = "Bearer "; + if !authorization.starts_with(bearer) { + Err(Status::internal("Invalid auth header!"))?; + } + let auth = authorization[bearer.len()..].to_string(); + + if let Some(context) = self.contexts.get(&auth) { + Ok(context.clone()) + } else { + Err(Status::internal(format!( + "Context handle not found: {auth}" + )))? + } + } + + fn get_plan(&self, handle: &str) -> Result { + if let Some(plan) = self.statements.get(handle) { + Ok(plan.clone()) + } else { + Err(Status::internal(format!("Plan handle not found: {handle}")))? + } + } + + fn get_result(&self, handle: &str) -> Result, Status> { + if let Some(result) = self.results.get(handle) { + Ok(result.clone()) + } else { + Err(Status::internal(format!( + "Request handle not found: {handle}" + )))? + } + } + + async fn tables(&self, ctx: Arc) -> RecordBatch { + let schema = Arc::new(Schema::new(vec![ + Field::new("catalog_name", DataType::Utf8, true), + Field::new("db_schema_name", DataType::Utf8, true), + Field::new("table_name", DataType::Utf8, false), + Field::new("table_type", DataType::Utf8, false), + ])); + + let mut catalogs = vec![]; + let mut schemas = vec![]; + let mut names = vec![]; + let mut types = vec![]; + for catalog in ctx.catalog_names() { + let catalog_provider = ctx.catalog(&catalog).unwrap(); + for schema in catalog_provider.schema_names() { + let schema_provider = catalog_provider.schema(&schema).unwrap(); + for table in schema_provider.table_names() { + let table_provider = schema_provider.table(&table).await.unwrap().unwrap(); + catalogs.push(catalog.clone()); + schemas.push(schema.clone()); + names.push(table.clone()); + types.push(table_provider.table_type().to_string()) + } + } + } + + RecordBatch::try_new( + schema, + [catalogs, schemas, names, types] + .into_iter() + .map(|i| Arc::new(StringArray::from(i)) as ArrayRef) + .collect::>(), + ) + .unwrap() + } + + fn remove_plan(&self, handle: &str) -> Result<(), Status> { + self.statements.remove(&handle.to_string()); + Ok(()) + } + + fn remove_result(&self, handle: &str) -> Result<(), Status> { + self.results.remove(&handle.to_string()); + Ok(()) + } +} + +#[tonic::async_trait] +impl FlightSqlService for FlightSqlServiceImpl { + type FlightService = FlightSqlServiceImpl; + + // This function will be triggered if client JDBC property's `user` and `password` field set + async fn do_handshake( + &self, + _request: Request>, + ) -> Result< + Response> + Send>>>, + Status, + > { + info!("do_handshake"); + if let Some(msg) = _request.metadata().get("create") { + // A new round start at SQLancer, clear the ctx + info!("Resetting ctx {:?}", msg); + let new_ctx = + SessionContext::new_with_config(SessionConfig::new().with_information_schema(true)); + + let mut ctx_guard = self.ctx.lock().await; // Use `lock()` for async Mutex + *ctx_guard = new_ctx; + } + // no authentication actually takes place here + // see Ballista implementation for example of basic auth + // in this case, we simply accept the connection and create a new SessionContext + // the SessionContext will be re-used within this same connection/session + let token = self.create_ctx().await?; + + let result = HandshakeResponse { + protocol_version: 0, + payload: token.as_bytes().to_vec().into(), + }; + let result = Ok(result); + let output = futures::stream::iter(vec![result]); + let str = format!("Bearer {token}"); + let mut resp: Response> + Send>>> = + Response::new(Box::pin(output)); + let md = MetadataValue::try_from(str) + .map_err(|_| Status::invalid_argument("authorization not parsable"))?; + resp.metadata_mut().insert("authorization", md); + Ok(resp) + } + + async fn do_get_fallback( + &self, + _request: Request, + message: Any, + ) -> Result::DoGetStream>, Status> { + if !message.is::() { + Err(Status::unimplemented(format!( + "do_get: The defined request is invalid: {}", + message.type_url + )))? + } + + let fr: FetchResults = message + .unpack() + .map_err(|e| Status::internal(format!("{e:?}")))? + .ok_or_else(|| Status::internal("Expected FetchResults but got None!"))?; + + let handle = fr.handle; + + info!("getting results for {handle}"); + let result = self.get_result(&handle)?; + // if we get an empty result, create an empty schema + let (schema, batches) = match result.first() { + None => (Arc::new(Schema::empty()), vec![]), + Some(batch) => (batch.schema(), result.clone()), + }; + + let batch_stream = futures::stream::iter(batches).map(Ok); + + let stream = FlightDataEncoderBuilder::new() + .with_schema(schema) + .build(batch_stream) + .map_err(Status::from); + + Ok(Response::new(Box::pin(stream))) + } + + async fn get_flight_info_prepared_statement( + &self, + cmd: CommandPreparedStatementQuery, + _request: Request, + ) -> Result, Status> { + info!("get_flight_info_prepared_statement {:?}", cmd); + let handle = std::str::from_utf8(&cmd.prepared_statement_handle) + .map_err(|e| status!("Unable to parse uuid", e))?; + + //let ctx = self.get_ctx(&request)?; + let plan = self.get_plan(handle)?; + + let ctx_guard = self.ctx.lock().await; + let state = (*ctx_guard).state(); + let df = DataFrame::new(state, plan); + let result = df + .collect() + .await + .map_err(|e| status!("Error executing query", e))?; + + // if we get an empty result, create an empty schema + let schema = match result.first() { + None => Schema::empty(), + Some(batch) => (*batch.schema()).clone(), + }; + + self.results.insert(handle.to_string(), result); + + // if we had multiple endpoints to connect to, we could use this Location + // but in the case of standalone DataFusion, we don't + // let loc = Location { + // uri: "grpc+tcp://127.0.0.1:50051".to_string(), + // }; + let fetch = FetchResults { + handle: handle.to_string(), + }; + let buf = fetch.as_any().encode_to_vec().into(); + let ticket = Ticket { ticket: buf }; + + let info = FlightInfo::new() + // Encode the Arrow schema + .try_with_schema(&schema) + .expect("encoding failed") + .with_endpoint(FlightEndpoint::new().with_ticket(ticket)) + .with_descriptor(FlightDescriptor { + r#type: DescriptorType::Cmd.into(), + cmd: Default::default(), + path: vec![], + }); + let resp = Response::new(info); + Ok(resp) + } + + async fn get_flight_info_tables( + &self, + _query: CommandGetTables, + request: Request, + ) -> Result, Status> { + info!("get_flight_info_tables"); + let ctx = self.get_ctx(&request)?; + let data = self.tables(ctx).await; + let schema = data.schema(); + + let uuid = Uuid::new_v4().hyphenated().to_string(); + self.results.insert(uuid.clone(), vec![data]); + + let fetch = FetchResults { handle: uuid }; + let buf = fetch.as_any().encode_to_vec().into(); + let ticket = Ticket { ticket: buf }; + + let info = FlightInfo::new() + // Encode the Arrow schema + .try_with_schema(&schema) + .expect("encoding failed") + .with_endpoint(FlightEndpoint::new().with_ticket(ticket)) + .with_descriptor(FlightDescriptor { + r#type: DescriptorType::Cmd.into(), + cmd: Default::default(), + path: vec![], + }); + let resp = Response::new(info); + Ok(resp) + } + + async fn do_put_prepared_statement_update( + &self, + handle: CommandPreparedStatementUpdate, + _request: Request, + ) -> Result { + info!("do_put_prepared_statement_update"); + // statements like "CREATE TABLE.." or "SET datafusion.nnn.." call this function + // and we are required to return some row count here + let handle = std::str::from_utf8(&handle.prepared_statement_handle) + .map_err(|e| status!("Unable to parse uuid", e))?; + + //let ctx = self.get_ctx(&request)?; + let plan = self.get_plan(handle)?; + //println!("do_put_prepared_statement_update plan is {:?}", plan); + + let ctx_guard = self.ctx.lock().await; + let state = (*ctx_guard).state(); + let df = DataFrame::new(state, plan); + df.collect() + .await + .map_err(|e| status!("Error executing query", e))?; + + Ok(1) + } + + async fn do_action_create_prepared_statement( + &self, + query: ActionCreatePreparedStatementRequest, + _request: Request, + ) -> Result { + let user_query = query.query.as_str(); + info!("do_action_create_prepared_statement: {user_query}"); + + //let ctx = self.get_ctx(&request)?; + + let ctx_guard = self.ctx.lock().await; + let plan = (*ctx_guard) + .sql(user_query) + .await + .and_then(|df| df.into_optimized_plan()) + .map_err(|e| Status::internal(format!("Error building plan: {e}")))?; + + // store a copy of the plan, it will be used for execution + let plan_uuid = Uuid::new_v4().hyphenated().to_string(); + self.statements.insert(plan_uuid.clone(), plan.clone()); + + let plan_schema = plan.schema(); + + let arrow_schema = (&**plan_schema).into(); + let message = SchemaAsIpc::new(&arrow_schema, &IpcWriteOptions::default()) + .try_into() + .map_err(|e| status!("Unable to serialize schema", e))?; + let IpcMessage(schema_bytes) = message; + + let res = ActionCreatePreparedStatementResult { + prepared_statement_handle: plan_uuid.into(), + dataset_schema: schema_bytes, + parameter_schema: Default::default(), + }; + Ok(res) + } + + async fn do_action_close_prepared_statement( + &self, + handle: ActionClosePreparedStatementRequest, + _request: Request, + ) -> Result<(), Status> { + info!("do_action_close_prepared_statement"); + let handle = std::str::from_utf8(&handle.prepared_statement_handle); + if let Ok(handle) = handle { + info!("do_action_close_prepared_statement: removing plan and results for {handle}"); + let _ = self.remove_plan(handle); + let _ = self.remove_result(handle); + } + Ok(()) + } + + async fn register_sql_info(&self, _id: i32, _result: &SqlInfo) {} +} + +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct FetchResults { + #[prost(string, tag = "1")] + pub handle: ::prost::alloc::string::String, +} + +impl ProstMessageExt for FetchResults { + fn type_url() -> &'static str { + "type.googleapis.com/datafusion.example.com.sql.FetchResults" + } + + fn as_any(&self) -> Any { + Any { + type_url: FetchResults::type_url().to_string(), + value: ::prost::Message::encode_to_vec(self).into(), + } + } +} diff --git a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java new file mode 100644 index 000000000..55f18fecd --- /dev/null +++ b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java @@ -0,0 +1,95 @@ +package sqlancer.datafusion.test; + +import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.ERROR; +import static sqlancer.datafusion.ast.DataFusionSelect.getRandomSelect; + +import java.sql.SQLException; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.datafusion.DataFusionErrors; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionToStringVisitor; +import sqlancer.datafusion.DataFusionUtil; +import sqlancer.datafusion.ast.DataFusionSelect; + +public class DataFusionNoRECOracle extends NoRECBase + implements TestOracle { + + private final DataFusionGlobalState state; + + public DataFusionNoRECOracle(DataFusionGlobalState globalState) { + super(globalState); + this.state = globalState; + DataFusionErrors.registerExpectedExecutionErrors(errors); + } + + /* + * Non-Optimizing Reference Engine Construction q1: SELECT [expr1] FROM [expr2] WHERE [expr3] q2: SELECT [expr3] + * FROM [expr2] + * + * Oracle Check: q1's result size equals to `true` count in q2's result set + */ + @Override + public void check() throws SQLException { + /* + * Setup Q1 and Q2 + */ + // generate a random: + // SELECT [expr1] FROM [expr2] WHERE [expr3] + DataFusionSelect randomSelect = getRandomSelect(state); + // Q1: SELECT count(*) FROM [expr2] WHERE [expr3] + DataFusionSelect q1 = new DataFusionSelect(); + q1.setFetchColumnsString("COUNT(*)"); + q1.setFromList(randomSelect.getFromList()); + q1.setWhereClause(randomSelect.getWhereClause()); + // Q2: SELECT count(case when [expr3] then 1 else null end) FROM [expr2] + DataFusionSelect q2 = new DataFusionSelect(); + String selectExpr = String.format("COUNT(CASE WHEN %S THEN 1 ELSE NULL END)", + DataFusionToStringVisitor.asString(randomSelect.getWhereClause())); + q2.setFetchColumnsString(selectExpr); + q2.setFromList(randomSelect.getFromList()); + q2.setWhereClause(null); + + /* + * Execute Q1 and Q2 + */ + String q1String = DataFusionToStringVisitor.asString(q1); + String q2String = DataFusionToStringVisitor.asString(q2); + List q1ResultSet = null; + List q2ResultSet = null; + try { + q1ResultSet = ComparatorHelper.getResultSetFirstColumnAsString(q1String, errors, state); + q2ResultSet = ComparatorHelper.getResultSetFirstColumnAsString(q2String, errors, state); + } catch (AssertionError e) { + // Append detailed error message + String replay = DataFusionUtil.getReplay(state.getDatabaseName()); + String newMessage = e.getMessage() + "\n" + e.getCause() + "\n" + replay + "\n"; + state.dfLogger.appendToLog(ERROR, newMessage); + + throw new AssertionError(newMessage); + } + + /* + * NoREC check + */ + int count1 = q1ResultSet != null ? Integer.parseInt(q1ResultSet.get(0)) : -1; + int count2 = q2ResultSet != null ? Integer.parseInt(q2ResultSet.get(0)) : -1; + if (count1 != count2) { + StringBuilder errorMessage = new StringBuilder().append("NoREC oracle violated:\n") + .append(" Q1(result size ").append(count1).append("):").append(q1String).append(";\n") + .append(" Q2(result size ").append(count2).append("):").append(q2String).append(";\n") + .append("=======================================\n").append("Reproducer: \n"); + + String replay = DataFusionUtil.getReplay(state.getDatabaseName()); + + String errorLog = errorMessage.toString() + replay + "\n"; + String indentedErrorLog = errorLog.replaceAll("(?m)^", " "); + state.dfLogger.appendToLog(ERROR, errorLog); + + throw new AssertionError("\n\n" + indentedErrorLog); + } + } +} diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java new file mode 100644 index 000000000..b304a4131 --- /dev/null +++ b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java @@ -0,0 +1,38 @@ +package sqlancer.datafusion.test; + +import java.sql.SQLException; + +import sqlancer.common.ast.newast.Node; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionSelect; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +public class DataFusionQueryPartitioningBase + extends TernaryLogicPartitioningOracleBase, DataFusionGlobalState> + implements TestOracle { + DataFusionGlobalState state; + DataFusionExpressionGenerator gen; + DataFusionSelect select; + + public DataFusionQueryPartitioningBase(DataFusionGlobalState state) { + super(state); + this.state = state; + } + + @Override + public void check() throws SQLException { + select = DataFusionSelect.getRandomSelect(state); + gen = select.exprGen; + initializeTernaryPredicateVariants(); + } + + @Override + protected ExpressionGenerator> getGen() { + return gen; + } + +} diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java new file mode 100644 index 000000000..3235c3d69 --- /dev/null +++ b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java @@ -0,0 +1,67 @@ +package sqlancer.datafusion.test; + +import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.ERROR; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.ComparatorHelper; +import sqlancer.datafusion.DataFusionErrors; +import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; +import sqlancer.datafusion.DataFusionToStringVisitor; +import sqlancer.datafusion.DataFusionUtil; +import sqlancer.datafusion.ast.DataFusionSelect; + +public class DataFusionQueryPartitioningWhereTester extends DataFusionQueryPartitioningBase { + public DataFusionQueryPartitioningWhereTester(DataFusionGlobalState state) { + super(state); + DataFusionErrors.registerExpectedExecutionErrors(errors); + } + + /* + * Query Partitioning - Where q: SELECT [expr1] FROM [expr2] qp1: SELECT [expr1] FROM [expr2] WHERE [expr3] qp2: + * SELECT [expr1] FROM [expr2] WHERE NOT [expr3] qp3: SELECT [expr1] FROM [expr2] WHERE [expr3] IS NULL + * + * Oracle check: q's result equals to union(qp1, qp2, qp3) + */ + @Override + public void check() throws SQLException { + // generate a random 'SELECT [expr1] FROM [expr2] WHERE [expr3] + super.check(); + DataFusionSelect randomSelect = select; + randomSelect.setWhereClause(null); + + // Construct q + String qString = DataFusionToStringVisitor.asString(randomSelect); + // Construct qp1, qp2, qp3 + randomSelect.setWhereClause(predicate); + String qp1String = DataFusionToStringVisitor.asString(randomSelect); + randomSelect.setWhereClause(negatedPredicate); + String qp2String = DataFusionToStringVisitor.asString(randomSelect); + randomSelect.setWhereClause(isNullPredicate); + String qp3String = DataFusionToStringVisitor.asString(randomSelect); + + try { + /* + * Run all queires + */ + List qResultSet = ComparatorHelper.getResultSetFirstColumnAsString(qString, errors, state); + List combinedString = new ArrayList<>(); + List qpResultSet = ComparatorHelper.getCombinedResultSet(qp1String, qp2String, qp3String, + combinedString, true, state, errors); + /* + * Query Partitioning-Where check + */ + ComparatorHelper.assumeResultSetsAreEqual(qResultSet, qpResultSet, qString, combinedString, state, + ComparatorHelper::canonicalizeResultValue); + } catch (AssertionError e) { + // Append more error message + String replay = DataFusionUtil.getReplay(state.getDatabaseName()); + String newMessage = e.getMessage() + "\n" + e.getCause() + "\n" + replay + "\n"; + state.dfLogger.appendToLog(ERROR, newMessage); + + throw new AssertionError(newMessage); + } + } +} diff --git a/test/sqlancer/dbms/TestDataFusion.java b/test/sqlancer/dbms/TestDataFusion.java new file mode 100644 index 000000000..ae568e40e --- /dev/null +++ b/test/sqlancer/dbms/TestDataFusion.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestDataFusion { + @Test + public void testDataFusion() { + String datafusionAvailable = System.getenv("DATAFUSION_AVAILABLE"); + boolean datafusionIsAvailable = datafusionAvailable != null && datafusionAvailable.equalsIgnoreCase("true"); + assumeTrue(datafusionIsAvailable); + + assertEquals(0, Main.executeMain("--random-seed", "0", "--num-threads", "1", // TODO(datafusion) update when + // multithread is supported + "--timeout-seconds", TestConfig.SECONDS, "--num-queries", TestConfig.NUM_QUERIES, "datafusion")); + } +} From 852c1e4fecc853dc6246b45a3cc3045da9d8e920 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Mon, 15 Jul 2024 14:18:44 +0200 Subject: [PATCH 452/856] [ClickHouse] Added pattern for changed error message (#955) * [ClickHouse] Added pattern for changed error message https://fiddle.clickhouse.com/a3a95024-4da7-4275-baff-86f116014744 ``` Received exception from server (version 24.2.3): Code: 403. DB::Exception: Received from localhost:9000. DB::Exception: Cannot get JOIN keys from JOIN ON section: '476505718 = `_--right_2.c0`', found keys: [Left keys: [] Right keys [] Condition columns: '', 'equals(476505718, _--right_2.c0)']. (INVALID_JOIN_ON_EXPRESSION) (query: SELECT SUM(check <> 0) FROM ((SELECT right_0.c0 AS `check` FROM t0 AS left FULL OUTER JOIN t0 AS right_0 ON ((left.c0)=(right_0.c0)) LEFT OUTER JOIN t0 AS right_1 ON ((left.c0)=(right_1.c0)) LEFT ANTI JOIN t0 AS right_2 ON ((476505718)=(right_2.c0)))) as res;) ``` * formatter --- src/sqlancer/clickhouse/ClickHouseErrors.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index c4ccd7e44..0354a6172 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -45,8 +45,8 @@ public static List getExpectedExpressionErrors() { "Positional argument numeric constant expression is not representable as", "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)", - "Unexpected inf or nan to integer conversion", "Unsigned type must not contain", - "Unexpected inf or nan to integer conversion", + "Cannot get JOIN keys from JOIN ON section", "Unexpected inf or nan to integer conversion", + "Unsigned type must not contain", "Unexpected inf or nan to integer conversion", // The way we generate JOINs we can have ambiguous left table column without // alias From ec00b2d19115dc9ee080835b65b53d6aa7a93848 Mon Sep 17 00:00:00 2001 From: Robins Date: Tue, 16 Jul 2024 11:01:01 +0930 Subject: [PATCH 453/856] Avoid storage parameters during CREATE TABLE for partitioned tables (#956) CREATE TABLE WITH() allows storage parameters, but partitioned tables emit an error (given below) if attempted. This is because partitioned non-leaf tables are virutal tables [1] and don't accept storage parameters. Sample Error: "ERROR: cannot specify storage parameters for a partitioned table" This patch skips storage parameter generation for partitioned tables. Ref: 1. https://www.postgresql.org/docs/current/ddl-partitioning.html --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 7d65b8ebd..9ae649ae5 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -113,7 +113,9 @@ private void createStandard() throws AssertionError { generateInherits(); generatePartitionBy(); generateUsing(); - PostgresCommon.generateWith(sb, globalState, errors); + if (!isPartitionedTable) { + PostgresCommon.generateWith(sb, globalState, errors); + } if (Randomly.getBoolean() && isTemporaryTable) { sb.append(" ON COMMIT "); sb.append(Randomly.fromOptions("PRESERVE ROWS", "DELETE ROWS", "DROP")); From 6d2b92a5b72a5904d5f7827df6ff9488668f55e6 Mon Sep 17 00:00:00 2001 From: Yongting You <2010youy01@gmail.com> Date: Tue, 16 Jul 2024 17:44:17 +0800 Subject: [PATCH 454/856] Review feedback --- .github/workflows/main.yml | 2 +- .../datafusion/DataFusionProvider.java | 7 +++--- src/sqlancer/datafusion/DataFusionUtil.java | 25 ++++++++----------- .../server/datafusion_server/Cargo.toml | 13 ++++------ .../server/datafusion_server/src/main.rs | 9 ++----- 5 files changed, 22 insertions(+), 34 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0f2a12c62..bd65c9890 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -214,7 +214,7 @@ jobs: - name: Run DataFusion Server run: | cd src/sqlancer/datafusion/server/datafusion_server - cargo run --features "datafusion_stable" & sleep 300 + cargo run & sleep 300 - name: Set up JDK 11 uses: actions/setup-java@v3 with: diff --git a/src/sqlancer/datafusion/DataFusionProvider.java b/src/sqlancer/datafusion/DataFusionProvider.java index 161c324dd..37328e4ba 100644 --- a/src/sqlancer/datafusion/DataFusionProvider.java +++ b/src/sqlancer/datafusion/DataFusionProvider.java @@ -1,7 +1,7 @@ package sqlancer.datafusion; -import static java.lang.System.exit; import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.DML; +import static sqlancer.datafusion.DataFusionUtil.dfAssert; import static sqlancer.datafusion.DataFusionUtil.displayTables; import java.sql.Connection; @@ -52,13 +52,12 @@ public void generateDatabase(DataFusionGlobalState globalState) throws Exception List allTables = globalState.getSchema().getDatabaseTables(); List allTablesName = allTables.stream().map(t -> t.getName()).collect(Collectors.toList()); if (allTablesName.isEmpty()) { - System.out.println("Generate database failed"); - exit(1); + dfAssert(false, "Generate Database failed."); } // Randomly insert some data into existing tables for (DataFusionTable table : allTables) { - int nInsertQuery = globalState.getRandomly().getInteger(0, 8); // [0, 10) + int nInsertQuery = globalState.getRandomly().getInteger(0, globalState.getOptions().getMaxNumberInserts()); for (int i = 0; i < nInsertQuery; i++) { SQLQueryAdapter insertQuery = null; diff --git a/src/sqlancer/datafusion/DataFusionUtil.java b/src/sqlancer/datafusion/DataFusionUtil.java index e37ad8d64..8761bec9b 100644 --- a/src/sqlancer/datafusion/DataFusionUtil.java +++ b/src/sqlancer/datafusion/DataFusionUtil.java @@ -1,7 +1,5 @@ package sqlancer.datafusion; -import static java.lang.System.exit; - import java.io.BufferedReader; import java.io.File; import java.io.FileReader; @@ -66,11 +64,15 @@ public static String displayTables(DataFusionGlobalState state, List fro return resultStringBuilder.toString(); } + // During development, you might want to manually let this function call exit(1) to fail fast public static void dfAssert(boolean condition, String message) { if (!condition) { - String methodName = Thread.currentThread().getStackTrace()[2].getMethodName(); - System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message); - exit(1); + // // Development mode assertion failure + // String methodName = Thread.currentThread().getStackTrace()[2]// .getMethodName(); + // System.err.println("DataFusion assertion failed in function '" + methodName + "': " + message); + // exit(1); + + throw new AssertionError(message); } } @@ -149,9 +151,7 @@ public void appendToLog(DataFusionLogType logType, String logContent) { try { logFileWriter = new FileWriter(errorLogFile, true); } catch (IOException e) { - System.out.println("Failed to create FileWriter for errorLogFIle"); - e.printStackTrace(); - exit(1); + dfAssert(false, "Failed to create FileWriter for errorLogFIle"); } DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); String formattedDateTime = LocalDateTime.now().format(formatter); @@ -175,14 +175,11 @@ public void appendToLog(DataFusionLogType logType, String logContent) { logFileWriter.write(logContent); logFileWriter.flush(); } catch (IOException e) { - System.out.println("Failed to write to " + logType + " log: " + e.getMessage()); - e.printStackTrace(); - exit(1); + String err = "Failed to write to " + logType + " log: " + e.getMessage(); + dfAssert(false, err); } } else { - System.out.println("appending to log failed"); - Thread.currentThread().getStackTrace(); - exit(1); + dfAssert(false, "appending to log failed"); } } diff --git a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml index ebff15b35..cd8b85e1d 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml +++ b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml @@ -18,8 +18,10 @@ async-trait = "0.1.73" bytes = "1.4" chrono = { version = "0.4.34", default-features = false } dashmap = "5.5.0" -datafusion_stable = { package = "datafusion", git = "https://github.com/apache/datafusion.git", rev = "e693ed7", optional = true } -datafusion_dev = { package = "datafusion", git = "https://github.com/apache/datafusion.git", branch = "main", optional = true } +# This version is for SQLancer CI run +datafusion = { version = "40.0.0" } +# Use following line if you want to test against the latest main branch of DataFusion +# datafusion = { git = "https://github.com/apache/datafusion.git", branch = "main" } env_logger = "0.11" futures = "0.3" half = { version = "2.2.1", default-features = false } @@ -41,9 +43,4 @@ mimalloc = { version = "0.1", default-features = false } [[bin]] name = "datafusion-server" -path = "src/main.rs" - -[features] -default = ["datafusion_stable_feature"] -datafusion_stable_feature = ["datafusion_stable"] -datafusion_dev_feature = ["datafusion_dev"] +path = "src/main.rs" \ No newline at end of file diff --git a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs index c19728bb6..13ec73e96 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs +++ b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs @@ -1,8 +1,3 @@ -#[cfg(feature = "datafusion_dev")] -extern crate datafusion_dev as datafusion; -#[cfg(feature = "datafusion_stable")] -extern crate datafusion_stable as datafusion; - use arrow::array::{ArrayRef, StringArray}; use arrow::ipc::writer::IpcWriteOptions; use arrow::record_batch::RecordBatch; @@ -21,8 +16,8 @@ use arrow_flight::{ }; use arrow_schema::{DataType, Field, Schema}; use dashmap::DashMap; -use datafusion_stable::logical_expr::LogicalPlan; -use datafusion_stable::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; +use datafusion::logical_expr::LogicalPlan; +use datafusion::prelude::{DataFrame, ParquetReadOptions, SessionConfig, SessionContext}; use futures::{Stream, StreamExt, TryStreamExt}; use log::info; use mimalloc::MiMalloc; From 0316c1c9c2fa2ad2a82977cfe2587faaba78cec8 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 17 Jul 2024 12:36:27 +0930 Subject: [PATCH 455/856] Ensure INHERITS() only uses tables (not views) In Postgres, CREATE TABLE INHERITS () throws an error when VIEWs are provided, sample given below. ERROR: inherited relation "pg_buffercache" is not a table or foreign table Ref: 1. https://www.postgresql.org/docs/current/ddl-inherit.html --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 9ae649ae5..29ccfcf2c 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -207,7 +207,7 @@ private void generateUsing() { } private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { + if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); From cc189b4d32961c09916a8870a0c0df3c5ce57c59 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 17 Jul 2024 12:40:42 +0930 Subject: [PATCH 456/856] Revert "Ensure INHERITS() only uses tables (not views)" This reverts commit 0316c1c9c2fa2ad2a82977cfe2587faaba78cec8. --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 29ccfcf2c..9ae649ae5 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -207,7 +207,7 @@ private void generateUsing() { } private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { + if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); From 96adb5446e05e52a8aff88b4aee7a5e42ff2c6a2 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 17 Jul 2024 12:42:23 +0930 Subject: [PATCH 457/856] Ensure INHERITS() only uses tables (not views) In Postgres, CREATE TABLE INHERITS () throws an error when VIEWs are provided, sample given below. ERROR: inherited relation "pg_buffercache" is not a table or foreign table Ref: 1. https://www.postgresql.org/docs/current/ddl-inherit.html --- src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 9ae649ae5..29ccfcf2c 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -207,7 +207,7 @@ private void generateUsing() { } private void generateInherits() { - if (Randomly.getBoolean() && !newSchema.getDatabaseTables().isEmpty()) { + if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) .collect(Collectors.joining(", "))); From 51e7ae87daba53699d1e9318f827b612d871f510 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:22:03 +0200 Subject: [PATCH 458/856] Another fix --- src/sqlancer/clickhouse/ClickHouseErrors.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 0354a6172..e6a4613d9 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -46,7 +46,8 @@ public static List getExpectedExpressionErrors() { "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", "with constants is not supported. (INVALID_JOIN_ON_EXPRESSION)", "Cannot get JOIN keys from JOIN ON section", "Unexpected inf or nan to integer conversion", - "Unsigned type must not contain", "Unexpected inf or nan to integer conversion", + "Cannot determine join keys in", "Unsigned type must not contain", + "Unexpected inf or nan to integer conversion", // The way we generate JOINs we can have ambiguous left table column without // alias From 34e46423efc7bbc24b9389d9b9ea99ffd2ae99fd Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Fri, 19 Jul 2024 17:22:17 +0200 Subject: [PATCH 459/856] Materialize: Use different database name (#959) Since materialize already exists by default and is owned by mz_system: org.postgresql.util.PSQLException: ERROR: must be owner of DATABASE materialize at sqlancer.qpg.materialize.TestMaterializeQueryPlan.testMaterializeQueryPlan(TestMaterializeQueryPlan.java:33) --- test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java index 08e402037..f282af42e 100644 --- a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java +++ b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java @@ -21,7 +21,7 @@ void testMaterializeQueryPlan() throws Exception { boolean materializeIsAvailable = materialize != null && materialize.equalsIgnoreCase("true"); assumeTrue(materializeIsAvailable); - String databaseName = "materialize"; + String databaseName = "queryplan"; MaterializeProvider provider = new MaterializeProvider(); MaterializeGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); MaterializeOptions materializeOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); @@ -42,7 +42,7 @@ void testMaterializeQueryPlan() throws Exception { String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); assertEquals( - "Return;Union;Get l0;Project (#2, #3, #0);Union;Negate;Project (#2);Get materialize.public.t2;Get materialize.public.t2;With;Get materialize.public.t1;Get materialize.public.t2;;Source materialize.public.t1;", + "Return // { arity: 3 };Union // { arity: 3 };Get l0 // { arity: 3 };Project (#2{c}, #3, #0) // { arity: 3 };Union // { arity: 1 };Negate // { arity: 1 };Project (#2) // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };With;ReadStorage queryplan.public.t1 // { arity: 2 };ReadStorage queryplan.public.t2 // { arity: 1 };;Source queryplan.public.t1;Source queryplan.public.t2;;Target cluster: quickstart;", queryPlan); } From 9c74a9527735cc62531f5e0ea0247222ee2f2027 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Tue, 20 Aug 2024 08:43:37 +0800 Subject: [PATCH 460/856] [Postgres] Use generic TLP where oracle (#943) * Implement TLPWhere generator for Postgres * Use generic TLP where oracle for Postgres --- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 31 ++++++++++- .../gen/PostgresExpressionGenerator.java | 26 +++++++-- .../gen/PostgresRandomQueryGenerator.java | 2 +- .../PostgresPivotedQuerySynthesisOracle.java | 2 +- .../tlp/PostgresTLPAggregateOracle.java | 2 +- .../postgres/oracle/tlp/PostgresTLPBase.java | 2 +- .../oracle/tlp/PostgresTLPHavingOracle.java | 2 +- .../oracle/tlp/PostgresTLPWhereOracle.java | 53 +++++++++---------- 8 files changed, 80 insertions(+), 40 deletions(-) diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java index 1ccecd0c5..dbfc26e4b 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java @@ -1,14 +1,20 @@ package sqlancer.citus.oracle.tlp; import java.sql.SQLException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; +import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.oracle.tlp.PostgresTLPBase; -public class CitusTLPWhereOracle extends PostgresTLPWhereOracle { +public class CitusTLPWhereOracle extends PostgresTLPBase { private final CitusTLPBase citusTLPBase; @@ -32,4 +38,25 @@ public void check() throws SQLException { whereCheck(); state.setDefaultAllowedFunctionTypes(); } + + void whereCheck() throws SQLException { + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBys()); + } + String originalQueryString = PostgresVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + select.setOrderByClauses(Collections.emptyList()); + select.setWhereClause(predicate); + String firstQueryString = PostgresVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = PostgresVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = PostgresVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state); + } } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 5b5851671..8c0c2352e 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -12,6 +12,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; @@ -64,7 +65,8 @@ import sqlancer.postgres.ast.PostgresSimilarTo; public class PostgresExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private final int maxDepth; @@ -108,7 +110,8 @@ public PostgresExpression generateExpression(int depth) { return generateExpression(depth, PostgresDataType.getRandomType()); } - public List generateOrderBy() { + @Override + public List generateOrderBys() { List orderBys = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber(); i++) { orderBys.add(new PostgresOrderByTerm(PostgresColumnValue.create(Randomly.fromList(columns), null), @@ -615,7 +618,7 @@ public static PostgresSubquery createSubquery(PostgresGlobalState globalState, S select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); @@ -691,6 +694,21 @@ public List getTableRefs() { .collect(Collectors.toList()); } + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBooleanWithRatherLowProbability()) { + return Arrays.asList(new PostgresColumnValue(PostgresColumn.createDummy("*"), null)); + } + allowAggregateFunctions = true; + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(columns); + for (PostgresColumn c : targetColumns) { + fetchColumns.add(new PostgresColumnValue(c, null)); + } + allowAggregateFunctions = false; + return fetchColumns; + } + @Override public String generateOptimizedQueryString(PostgresSelect select, PostgresExpression whereCondition, boolean shouldUseAggregate) { @@ -703,7 +721,7 @@ public String generateOptimizedQueryString(PostgresSelect select, PostgresExpres } select.setWhereClause(whereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(generateOrderBy()); + select.setOrderByClauses(generateOrderBys()); } select.setSelectType(SelectType.ALL); return select.asString(); diff --git a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java index 7fc5049cd..64d6278de 100644 --- a/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresRandomQueryGenerator.java @@ -45,7 +45,7 @@ public static PostgresSelect createRandomQuery(int nrColumns, PostgresGlobalStat } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java index c64f764e3..9db5365a3 100644 --- a/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresPivotedQuerySynthesisOracle.java @@ -63,7 +63,7 @@ public SQLQueryAdapter getRectifiedQuery() throws SQLException { selectStatement.setOffsetClause(offsetClause); } List orderBy = new PostgresExpressionGenerator(globalState).setColumns(columns) - .generateOrderBy(); + .generateOrderBys(); selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(PostgresVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java index a1ae5597e..dba8c0a6d 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPAggregateOracle.java @@ -61,7 +61,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = PostgresVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 2e34b1f87..77c4fb022 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -103,7 +103,7 @@ public static PostgresSubquery createSubquery(PostgresGlobalState globalState, S select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(PostgresConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java index 6af683903..2f520bebb 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPHavingOracle.java @@ -36,7 +36,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setHavingClause(predicate); String firstQueryString = PostgresVisitor.asString(select); diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java index d8aeb4938..3c183457d 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java @@ -1,45 +1,40 @@ package sqlancer.postgres.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class PostgresTLPWhereOracle extends PostgresTLPBase { +public class PostgresTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public PostgresTLPWhereOracle(PostgresGlobalState state) { - super(state); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - whereCheck(); + oracle.check(); } - protected void whereCheck() throws SQLException { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - String originalQueryString = PostgresVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = PostgresVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = PostgresVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = PostgresVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From 417b85a1e1dbfff7ddf30a8b183a038425259b88 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 11:08:07 +0800 Subject: [PATCH 461/856] [MariaDB] Implement expression interfaces --- .../mariadb/ast/MariaDBExpression.java | 5 ++++- src/sqlancer/mariadb/ast/MariaDBJoin.java | 5 ++++- .../mariadb/ast/MariaDBSelectStatement.java | 21 ++++++++++++++++++- 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/mariadb/ast/MariaDBExpression.java b/src/sqlancer/mariadb/ast/MariaDBExpression.java index 5caec1b40..d57c14888 100644 --- a/src/sqlancer/mariadb/ast/MariaDBExpression.java +++ b/src/sqlancer/mariadb/ast/MariaDBExpression.java @@ -1,5 +1,8 @@ package sqlancer.mariadb.ast; -public interface MariaDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; + +public interface MariaDBExpression extends Expression { } diff --git a/src/sqlancer/mariadb/ast/MariaDBJoin.java b/src/sqlancer/mariadb/ast/MariaDBJoin.java index b8110b7f2..968b8b29c 100644 --- a/src/sqlancer/mariadb/ast/MariaDBJoin.java +++ b/src/sqlancer/mariadb/ast/MariaDBJoin.java @@ -5,12 +5,13 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; -public class MariaDBJoin implements MariaDBExpression { +public class MariaDBJoin implements MariaDBExpression, Join { public enum JoinType { NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; @@ -36,6 +37,7 @@ public MariaDBTable getTable() { return table; } + @Override public MariaDBExpression getOnClause() { return onClause; } @@ -44,6 +46,7 @@ public JoinType getType() { return type; } + @Override public void setOnClause(MariaDBExpression onClause) { this.onClause = onClause; } diff --git a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java index 813b92337..f09e7051f 100644 --- a/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java +++ b/src/sqlancer/mariadb/ast/MariaDBSelectStatement.java @@ -4,8 +4,12 @@ import java.util.List; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -public class MariaDBSelectStatement extends SelectBase implements MariaDBExpression { +public class MariaDBSelectStatement extends SelectBase + implements MariaDBExpression, Select { public enum MariaDBSelectType { ALL, DISTINCT, DISTINCTROW; @@ -13,6 +17,7 @@ public enum MariaDBSelectType { private List groupBys = new ArrayList<>(); private List columns = new ArrayList<>(); + private List joinClauses = new ArrayList<>(); private MariaDBSelectType selectType = MariaDBSelectType.ALL; private MariaDBExpression whereCondition; @@ -52,4 +57,18 @@ public MariaDBExpression getWhereCondition() { return whereCondition; } + @Override + public List getJoinClauses() { + return joinClauses; + } + + @Override + public void setJoinClauses(List joinClauses) { + this.joinClauses = joinClauses; + } + + @Override + public String asString() { + return MariaDBVisitor.asString(this); + } } From 9ffe85beae597a111b95199fb48ccdb9e2ab4ffb Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 13:17:51 +0800 Subject: [PATCH 462/856] [MariaDB] Implement NoREC expression generator interface --- src/sqlancer/mariadb/MariaDBSchema.java | 4 + .../gen/MariaDBExpressionGenerator.java | 84 ++++++++++++++++--- .../mariadb/oracle/MariaDBDQPOracle.java | 3 +- 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index d1bcfbd97..808653b25 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -55,6 +55,10 @@ public boolean isPrimaryKey() { return isPrimaryKey; } + public static MariaDBColumn createDummy(String name) { + return new MariaDBColumn(name, MariaDBDataType.INT, false, 1); + } + } public static class MariaDBTables { diff --git a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java index 062bf509c..e769e399f 100644 --- a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java @@ -5,11 +5,14 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.StateToReproduce; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.mariadb.MariaDBProvider; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; +import sqlancer.mariadb.ast.MariaDBAggregate; +import sqlancer.mariadb.ast.MariaDBAggregate.MariaDBAggregateFunction; import sqlancer.mariadb.ast.MariaDBBinaryOperator; import sqlancer.mariadb.ast.MariaDBBinaryOperator.MariaDBBinaryComparisonOperator; import sqlancer.mariadb.ast.MariaDBColumnName; @@ -18,14 +21,21 @@ import sqlancer.mariadb.ast.MariaDBFunction; import sqlancer.mariadb.ast.MariaDBFunctionName; import sqlancer.mariadb.ast.MariaDBInOperation; +import sqlancer.mariadb.ast.MariaDBJoin; import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation; import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation.MariaDBPostfixUnaryOperator; +import sqlancer.mariadb.ast.MariaDBSelectStatement; +import sqlancer.mariadb.ast.MariaDBSelectStatement.MariaDBSelectType; +import sqlancer.mariadb.ast.MariaDBTableReference; +import sqlancer.mariadb.ast.MariaDBText; import sqlancer.mariadb.ast.MariaDBUnaryPrefixOperation; import sqlancer.mariadb.ast.MariaDBUnaryPrefixOperation.MariaDBUnaryPrefixOperator; -public class MariaDBExpressionGenerator { +public class MariaDBExpressionGenerator + implements NoRECGenerator { private final Randomly r; + private List targetTables = new ArrayList<>(); private List columns = new ArrayList<>(); public MariaDBExpressionGenerator(Randomly r) { @@ -66,14 +76,6 @@ public MariaDBExpressionGenerator setColumns(List columns) { return this; } - public MariaDBExpressionGenerator setCon(SQLConnection con) { - return this; - } - - public MariaDBExpressionGenerator setState(StateToReproduce state) { - return this; - } - private enum ExpressionType { LITERAL, COLUMN, BINARY_COMPARISON, UNARY_POSTFIX_OPERATOR, UNARY_PREFIX_OPERATOR, FUNCTION, IN } @@ -146,4 +148,64 @@ public MariaDBExpression getRandomExpression() { return getRandomExpression(0); } + @Override + public MariaDBExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + this.targetTables = targetTables.getTables(); + this.columns = targetTables.getColumns(); + return this; + } + + @Override + public List getTableRefs() { + List tableRefs = new ArrayList<>(); + for (MariaDBTable t : targetTables) { + MariaDBTableReference tableRef = new MariaDBTableReference(t); + tableRefs.add(tableRef); + } + return tableRefs; + } + + @Override + public MariaDBExpression generateBooleanExpression() { + return getRandomExpression(); + } + + @Override + public MariaDBSelectStatement generateSelect() { + return new MariaDBSelectStatement(); + } + + @Override + public List getRandomJoinClauses() { + return new ArrayList<>(); + } + + @Override + public String generateOptimizedQueryString(MariaDBSelectStatement select, MariaDBExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + MariaDBAggregate aggr = new MariaDBAggregate( + new MariaDBColumnName(new MariaDBColumn("*", MariaDBDataType.INT, false, 0)), + MariaDBAggregateFunction.COUNT); + select.setFetchColumns(Arrays.asList(aggr)); + } else { + MariaDBColumnName aggr = new MariaDBColumnName(MariaDBColumn.createDummy("*")); + select.setFetchColumns(Arrays.asList(aggr)); + } + + select.setWhereClause(whereCondition); + select.setSelectType(MariaDBSelectType.ALL); + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(MariaDBSelectStatement select, MariaDBExpression whereCondition) { + MariaDBPostfixUnaryOperation isTrue = new MariaDBPostfixUnaryOperation(MariaDBPostfixUnaryOperator.IS_TRUE, + whereCondition); + MariaDBText asText = new MariaDBText(isTrue, " as count", false); + select.setFetchColumns(Arrays.asList(asText)); + select.setSelectType(MariaDBSelectType.ALL); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as asdf"; + } } diff --git a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java index ee85f2df7..931b0ad1d 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java @@ -38,8 +38,7 @@ public MariaDBDQPOracle(MariaDBGlobalState globalState) { @Override public void check() throws Exception { MariaDBTables tables = s.getRandomTableNonEmptyTables(); - gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(tables.getColumns()) - .setCon(state.getConnection()).setState(state.getState()); + gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(tables.getColumns()); List fetchColumns = new ArrayList<>(); fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new MariaDBColumnName(c)) From 6e10142ca30d7134edb38685fcd1ed0537599382 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 13:35:10 +0800 Subject: [PATCH 463/856] [MariaDB] Use generic NoREC oracle --- .../mariadb/oracle/MariaDBNoRECOracle.java | 120 +++--------------- 1 file changed, 20 insertions(+), 100 deletions(-) diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index ab6438275..22c1eea23 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -1,126 +1,46 @@ package sqlancer.mariadb.oracle; import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import sqlancer.IgnoreMeException; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; -import sqlancer.mariadb.MariaDBSchema.MariaDBDataType; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -import sqlancer.mariadb.ast.MariaDBAggregate; -import sqlancer.mariadb.ast.MariaDBAggregate.MariaDBAggregateFunction; -import sqlancer.mariadb.ast.MariaDBColumnName; import sqlancer.mariadb.ast.MariaDBExpression; -import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation; -import sqlancer.mariadb.ast.MariaDBPostfixUnaryOperation.MariaDBPostfixUnaryOperator; +import sqlancer.mariadb.ast.MariaDBJoin; import sqlancer.mariadb.ast.MariaDBSelectStatement; -import sqlancer.mariadb.ast.MariaDBSelectStatement.MariaDBSelectType; -import sqlancer.mariadb.ast.MariaDBTableReference; -import sqlancer.mariadb.ast.MariaDBText; -import sqlancer.mariadb.ast.MariaDBVisitor; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; -public class MariaDBNoRECOracle extends NoRECBase implements TestOracle { +public class MariaDBNoRECOracle implements TestOracle { - private final MariaDBSchema s; - private static final int NOT_FOUND = -1; + NoRECOracle oracle; public MariaDBNoRECOracle(MariaDBGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - errors.add("is out of range"); - // regex - errors.add("unmatched parentheses"); - errors.add("nothing to repeat at offset"); - errors.add("missing )"); - errors.add("missing terminating ]"); - errors.add("range out of order in character class"); - errors.add("unrecognized character after "); - errors.add("Got error '(*VERB) not recognized or malformed"); - errors.add("must be followed by"); - errors.add("malformed number or name after"); - errors.add("digit expected after"); + MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(globalState.getRandomly()); + ExpectedErrors errors = ExpectedErrors.newErrors().with("is out of range").with("unmatched parentheses") + .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") + .with("range out of order in character class").with("unrecognized character after ") + .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") + .with("malformed number or name after").with("digit expected after").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - MariaDBTable randomTable = s.getRandomTable(); - List columns = randomTable.getColumns(); - MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(state.getRandomly()).setColumns(columns) - .setCon(con).setState(state.getState()); - MariaDBExpression randomWhereCondition = gen.getRandomExpression(); - List groupBys = Collections.emptyList(); // getRandomExpressions(columns); - int optimizedCount = getOptimizedQuery(randomTable, randomWhereCondition, groupBys); - int unoptimizedCount = getUnoptimizedQuery(randomTable, randomWhereCondition, groupBys); - if (optimizedCount == NOT_FOUND || unoptimizedCount == NOT_FOUND) { - throw new IgnoreMeException(); - } - if (optimizedCount != unoptimizedCount) { - state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); - throw new AssertionError(optimizedCount + " " + unoptimizedCount); - } + oracle.check(); } - private int getUnoptimizedQuery(MariaDBTable randomTable, MariaDBExpression randomWhereCondition, - List groupBys) throws SQLException { - MariaDBSelectStatement select = new MariaDBSelectStatement(); - select.setGroupByClause(groupBys); - MariaDBPostfixUnaryOperation isTrue = new MariaDBPostfixUnaryOperation(MariaDBPostfixUnaryOperator.IS_TRUE, - randomWhereCondition); - MariaDBText asText = new MariaDBText(isTrue, " as count", false); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(Arrays.asList(new MariaDBTableReference(randomTable))); - select.setSelectType(MariaDBSelectType.ALL); - int secondCount = 0; - - unoptimizedQueryString = "SELECT SUM(count) FROM (" + MariaDBVisitor.asString(select) + ") as asdf"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - try (SQLancerResultSet rs = q.executeAndGet(state)) { - if (rs == null) { - return NOT_FOUND; - } else { - while (rs.next()) { - secondCount = rs.getInt(1); - } - } - } - - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getOptimizedQuery(MariaDBTable randomTable, MariaDBExpression randomWhereCondition, - List groupBys) throws SQLException { - MariaDBSelectStatement select = new MariaDBSelectStatement(); - select.setGroupByClause(groupBys); - MariaDBAggregate aggr = new MariaDBAggregate( - new MariaDBColumnName(new MariaDBColumn("*", MariaDBDataType.INT, false, 0)), - MariaDBAggregateFunction.COUNT); - select.setFetchColumns(Arrays.asList(aggr)); - select.setFromList(Arrays.asList(new MariaDBTableReference(randomTable))); - select.setWhereClause(randomWhereCondition); - select.setSelectType(MariaDBSelectType.ALL); - int firstCount; - optimizedQueryString = MariaDBVisitor.asString(select); - SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - try (SQLancerResultSet rs = q.executeAndGet(state)) { - if (rs == null) { - firstCount = NOT_FOUND; - } else { - rs.next(); - firstCount = rs.getInt(1); - } - } catch (Exception e) { - throw new AssertionError(optimizedQueryString, e); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From d6695ca6406495b7d1e6ba020ff48fcdc825aa38 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 14:04:54 +0800 Subject: [PATCH 464/856] [MariaDB] Update tests --- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestMariaDB.java | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 67c406fba..084de35e7 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -6,6 +6,7 @@ public class TestConfig { public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; + public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { diff --git a/test/sqlancer/dbms/TestMariaDB.java b/test/sqlancer/dbms/TestMariaDB.java index f1b86a339..eb26bfb4a 100644 --- a/test/sqlancer/dbms/TestMariaDB.java +++ b/test/sqlancer/dbms/TestMariaDB.java @@ -39,11 +39,9 @@ public void testSelectAsString() { @Test public void testMariaDB() { - String mariaDBAvailable = System.getenv("MARIADB_AVAILABLE"); - boolean mariaDBIsAvailable = mariaDBAvailable != null && mariaDBAvailable.equalsIgnoreCase("true"); - assumeTrue(mariaDBIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MARIADB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-queries", "0", "mariadb" })); + "--num-queries", TestConfig.NUM_QUERIES, "mariadb" })); } } From f216c4187f12bed46a1ae98d39ee8235197ef065 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 20 Aug 2024 17:29:30 +0800 Subject: [PATCH 465/856] [MariaDB] Add join clause generation --- src/sqlancer/mariadb/ast/MariaDBJoin.java | 6 ++---- src/sqlancer/mariadb/ast/MariaDBStringVisitor.java | 2 +- .../mariadb/gen/MariaDBExpressionGenerator.java | 2 +- src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java | 4 ++-- src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java | 12 +++++++----- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/mariadb/ast/MariaDBJoin.java b/src/sqlancer/mariadb/ast/MariaDBJoin.java index 968b8b29c..d846f4605 100644 --- a/src/sqlancer/mariadb/ast/MariaDBJoin.java +++ b/src/sqlancer/mariadb/ast/MariaDBJoin.java @@ -6,7 +6,6 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.Join; -import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.gen.MariaDBExpressionGenerator; @@ -55,7 +54,7 @@ public void setType(JoinType type) { this.type = type; } - public static List getRandomJoinClauses(List tables, MariaDBGlobalState globalState) { + public static List getRandomJoinClauses(List tables, Randomly r) { List joinStatements = new ArrayList<>(); List options = new ArrayList<>(Arrays.asList(JoinType.values())); List columns = new ArrayList<>(); @@ -71,8 +70,7 @@ public static List getRandomJoinClauses(List tables, MariaDBTable table = Randomly.fromList(tables); tables.remove(table); columns.addAll(table.getColumns()); - MariaDBExpressionGenerator joinGen = new MariaDBExpressionGenerator(globalState.getRandomly()) - .setColumns(columns); + MariaDBExpressionGenerator joinGen = new MariaDBExpressionGenerator(r).setColumns(columns); MariaDBExpression joinClause = joinGen.getRandomExpression(); JoinType selectedOption = Randomly.fromList(options); if (selectedOption == JoinType.NATURAL) { diff --git a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java index 17c84ef4a..e1dbf476f 100644 --- a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java @@ -47,7 +47,7 @@ public void visit(MariaDBSelectStatement s) { } visit(s.getFromList().get(j)); } - for (MariaDBExpression j : s.getJoinList()) { + for (MariaDBExpression j : s.getJoinClauses()) { visit(j); } if (s.getWhereCondition() != null) { diff --git a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java index e769e399f..3b7386121 100644 --- a/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBExpressionGenerator.java @@ -177,7 +177,7 @@ public MariaDBSelectStatement generateSelect() { @Override public List getRandomJoinClauses() { - return new ArrayList<>(); + return MariaDBJoin.getRandomJoinClauses(targetTables, r); } @Override diff --git a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java index 931b0ad1d..dcd458193 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBDQPOracle.java @@ -56,8 +56,8 @@ public void check() throws Exception { } // Set the join. - List joinExpressions = MariaDBJoin.getRandomJoinClauses(tables.getTables(), state); - select.setJoinList(joinExpressions.stream().map(j -> (MariaDBExpression) j).collect(Collectors.toList())); + List joinExpressions = MariaDBJoin.getRandomJoinClauses(tables.getTables(), state.getRandomly()); + select.setJoinClauses(joinExpressions); // Set the from clause from the tables that are not used in the join. select.setFromList( diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java index 22c1eea23..039abd85f 100644 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java @@ -6,6 +6,7 @@ import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; +import sqlancer.mariadb.MariaDBErrors; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; @@ -21,11 +22,12 @@ public class MariaDBNoRECOracle implements TestOracle { public MariaDBNoRECOracle(MariaDBGlobalState globalState) { MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(globalState.getRandomly()); - ExpectedErrors errors = ExpectedErrors.newErrors().with("is out of range").with("unmatched parentheses") - .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") - .with("range out of order in character class").with("unrecognized character after ") - .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") - .with("malformed number or name after").with("digit expected after").build(); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MariaDBErrors.getCommonErrors()).with("is out of range") + .with("unmatched parentheses").with("nothing to repeat at offset").with("missing )") + .with("missing terminating ]").with("range out of order in character class") + .with("unrecognized character after ").with("Got error '(*VERB) not recognized or malformed") + .with("must be followed by").with("malformed number or name after").with("digit expected after") + .with("Could not create a join buffer").build(); this.oracle = new NoRECOracle<>(globalState, gen, errors); } From a306672cca6f9ce86e0cb541d4156f59921a6484 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Thu, 22 Aug 2024 10:46:30 +0800 Subject: [PATCH 466/856] [Citus] Disconnect other sessions before dropping database (#960) * [Citus] Disconnect other sessions before dropping database * Format code * [Citus] Add comment for terminate query --- src/sqlancer/citus/CitusProvider.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index c897d503e..747f72216 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -371,6 +371,10 @@ private void prepareCitusWorkerNodes(PostgresGlobalState globalState, List pg_backend_pid()"); s.execute("DROP DATABASE IF EXISTS " + databaseName); } try (Statement s = con.createStatement()) { From a05b0a025333368eb172a76561f9d114ec2da34b Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Fri, 23 Aug 2024 16:25:13 +0800 Subject: [PATCH 467/856] [StoneDB] Remove implementation (#963) --- .github/workflows/main.yml | 29 -- src/check_names.py | 1 - src/sqlancer/Main.java | 2 - src/sqlancer/stonedb/StoneDBBugs.java | 51 -- src/sqlancer/stonedb/StoneDBErrors.java | 63 --- src/sqlancer/stonedb/StoneDBOptions.java | 99 ---- src/sqlancer/stonedb/StoneDBProvider.java | 157 ------- src/sqlancer/stonedb/StoneDBSchema.java | 441 ------------------ .../stonedb/StoneDBToStringVisitor.java | 101 ---- .../stonedb/ast/StoneDBAdvancedFunction.java | 34 -- .../stonedb/ast/StoneDBAggregate.java | 34 -- src/sqlancer/stonedb/ast/StoneDBConstant.java | 211 --------- .../stonedb/ast/StoneDBExpression.java | 4 - src/sqlancer/stonedb/ast/StoneDBJoin.java | 126 ----- src/sqlancer/stonedb/ast/StoneDBSelect.java | 17 - .../gen/StoneDBExpressionGenerator.java | 340 -------------- .../gen/StoneDBIndexCreateGenerator.java | 121 ----- .../gen/StoneDBIndexDropGenerator.java | 47 -- .../gen/StoneDBTableAlterGenerator.java | 141 ------ .../gen/StoneDBTableCreateGenerator.java | 266 ----------- .../gen/StoneDBTableDeleteGenerator.java | 70 --- .../gen/StoneDBTableInsertGenerator.java | 142 ------ .../gen/StoneDBTableUpdateGenerator.java | 71 --- .../gen/StoneDBViewCreateGenerator.java | 78 ---- .../oracle/StoneDBAggregateOracle.java | 170 ------- .../stonedb/oracle/StoneDBFuzzOracle.java | 67 --- .../stonedb/oracle/StoneDBNoRECOracle.java | 134 ------ .../oracle/StoneDBQueryPartitioningBase.java | 71 --- ...toneDBQueryPartitioningDistinctTester.java | 42 -- ...StoneDBQueryPartitioningGroupByTester.java | 50 -- .../StoneDBQueryPartitioningHavingTester.java | 47 -- .../StoneDBQueryPartitioningWhereTester.java | 42 -- test/sqlancer/dbms/TestStoneDBFuzz.java | 21 - test/sqlancer/dbms/TestStoneDBNoRec.java | 22 - test/sqlancer/dbms/TestStoneDBTLP.java | 21 - 35 files changed, 3333 deletions(-) delete mode 100644 src/sqlancer/stonedb/StoneDBBugs.java delete mode 100644 src/sqlancer/stonedb/StoneDBErrors.java delete mode 100644 src/sqlancer/stonedb/StoneDBOptions.java delete mode 100644 src/sqlancer/stonedb/StoneDBProvider.java delete mode 100644 src/sqlancer/stonedb/StoneDBSchema.java delete mode 100644 src/sqlancer/stonedb/StoneDBToStringVisitor.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBAggregate.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBConstant.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBExpression.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBJoin.java delete mode 100644 src/sqlancer/stonedb/ast/StoneDBSelect.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java delete mode 100644 src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java delete mode 100644 src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java delete mode 100644 test/sqlancer/dbms/TestStoneDBFuzz.java delete mode 100644 test/sqlancer/dbms/TestStoneDBNoRec.java delete mode 100644 test/sqlancer/dbms/TestStoneDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 56bca0594..68b02c443 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -451,35 +451,6 @@ jobs: run: | mvn -Dtest=TestSQLiteQPG test - stonedb: - name: DBMS Tests (StoneDB) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Set up StoneDB - run: | - docker run -p 3306:3306 -itd -e MYSQL_ROOT_PASSWORD='123456' stoneatom/stonedb - ContainerID=$(docker ps --filter ancestor=stoneatom/stonedb --format "{{.ID}}") - docker exec $ContainerID bash - sleep 60s - docker exec $ContainerID sed -i "s/tianmu_insert_delayed=1/tianmu_insert_delayed=0/" /opt/stonedb57/install/my.cnf - docker exec $ContainerID /opt/stonedb57/install/bin/mysql -uroot -p123456 -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - sleep 60s - - name: Set up JDK 11 - uses: actions/setup-java@v3 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - - name: Build - run: mvn -B package -DskipTests=true - - name: StoneDB Tests - run: | - STONEDB_AVAILABLE=true mvn test -Dtest=TestStoneDBFuzz - tidb: name: DBMS Tests (TiDB) runs-on: ubuntu-latest diff --git a/src/check_names.py b/src/check_names.py index 41694e060..f2ab346c6 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -53,5 +53,4 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files["TiDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "tidb")) name_to_files["Y"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "yugabyte")) # has both YCQL and YSQL prefixes name_to_files["Doris"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "doris")) - name_to_files["StoneDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "stonedb")) verify_all_dbs(name_to_files) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 1cbc02645..5ffc3e94c 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -43,7 +43,6 @@ import sqlancer.presto.PrestoProvider; import sqlancer.questdb.QuestDBProvider; import sqlancer.sqlite3.SQLite3Provider; -import sqlancer.stonedb.StoneDBProvider; import sqlancer.tidb.TiDBProvider; import sqlancer.timescaledb.TimescaleDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; @@ -746,7 +745,6 @@ private static void checkForIssue799(List> providers) providers.add(new PostgresProvider()); providers.add(new QuestDBProvider()); providers.add(new SQLite3Provider()); - providers.add(new StoneDBProvider()); providers.add(new TiDBProvider()); providers.add(new TimescaleDBProvider()); providers.add(new YCQLProvider()); diff --git a/src/sqlancer/stonedb/StoneDBBugs.java b/src/sqlancer/stonedb/StoneDBBugs.java deleted file mode 100644 index 4864c56e5..000000000 --- a/src/sqlancer/stonedb/StoneDBBugs.java +++ /dev/null @@ -1,51 +0,0 @@ -package sqlancer.stonedb; - -public final class StoneDBBugs { - // https://github.com/stoneatom/stonedb/issues/1933 - public static boolean bug1933 = true; - // https://github.com/stoneatom/stonedb/issues/1942 - public static boolean bug1942 = true; - // https://github.com/stoneatom/stonedb/issues/1953 - public static boolean bug1953 = true; - // https://github.com/stoneatom/stonedb/issues/1945 - public static boolean bug1945 = true; - // CREATE TABLE t0(c0 INT); - // INSERT IGNORE INTO t0(c0) VALUE (DEFAULT); - // SELECT t0.c0 FROM t0 WHERE 0.4; -- expect 1 but got 0 - public static boolean bugNotReported1 = true; - // DELETE statements will result into crash, for example - // DELETE LOW_PRIORITY FROM t0; - // DELETE QUICK IGNORE FROM t0 WHERE -1370759901; - public static boolean bugNotReported2 = true; - // CREATE TABLE t0(c0 INT) ; - // INSERT INTO t0(c0) VALUES (DEFAULT); - // SELECT t0.c0 FROM t0 WHERE (('OC')>=(((t0.c0) IS NULL))); -- expected empty set but got 1 row - public static boolean bugNotReported3 = true; - // CREATE TABLE t0(c0 CHAR PRIMARY KEY); - // INSERT IGNORE INTO t0(c0) VALUE (DEFAULT); - // SELECT t0.c0 FROM t0 WHERE (t0.c0 BETWEEN (0) AND t0.c0); -- 1 - public static boolean bugNotReported4 = true; - // CREATE TABLE t0(c0 INT); - // INSERT INTO t0(c0) VALUE (DEFAULT); - // SELECT * FROM t0 WHERE (('00')>((('')AND(t0.c0)))); -- expected empty set but got 1 row - public static boolean bugNotReported5 = true; - // CREATE TABLE t0(c0 INT) ; - // INSERT INTO t0(c0) VALUE (100); - // SELECT * FROM t0 WHERE (('Xg')>(t0.c0)); -- expected empty set but got 1 row - // SELECT (('Xg')>(t0.c0)) FROM t0; - public static boolean bugNotReported6 = true; - // CREATE TABLE t0(c0 VARCHAR(1000)); - // INSERT INTO t0(c0) VALUE ('GX9vartD'); - // INSERT INTO t0(c0) VALUE ('16'); - // SELECT t0.c0 FROM t0 WHERE t0.c0; -- expected 1 row but got 2 rows - public static boolean bugNotReported7 = true; - // CREATE TABLE t0(c0 INT) ; - // SELECT * FROM t0 WHERE ((t0.c0)XOR((t0.c0))); - // ERROR 6 (HY000): The query includes syntax that is not supported by the storage engine. Either restructure the - // query with supported syntax, or enable the MySQL core::Query Path in config file to execute the query with - // reduced performance. - public static boolean bugNotReportedXOR = true; - - private StoneDBBugs() { - } -} diff --git a/src/sqlancer/stonedb/StoneDBErrors.java b/src/sqlancer/stonedb/StoneDBErrors.java deleted file mode 100644 index d8b0fd416..000000000 --- a/src/sqlancer/stonedb/StoneDBErrors.java +++ /dev/null @@ -1,63 +0,0 @@ -package sqlancer.stonedb; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.common.query.ExpectedErrors; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; - -public final class StoneDBErrors { - private StoneDBErrors() { - } - - public static List getExpectedExpressionErrors(StoneDBGlobalState globalState) { - ArrayList errors = new ArrayList<>(); - - // java.sql.SQLException: Incorrect DATE value: '292269055-12-02' - errors.add("Incorrect DATE value: "); - // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for - // column 'c1' at row 1 - errors.add("Incorrect string value: "); - // java.sql.SQLException: Incorrect integer value: 'ST' for column 'c1' at row 1 - errors.add("Incorrect integer value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Truncated - // incorrect INTEGER value: '#Q' - errors.add("Data truncation: Truncated incorrect INTEGER value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT - // value is out of range in - // '-((`database0`.`t0`.`c1` >> (not(`database0`.`t0`.`c1`))))' - errors.add("Data truncation: BIGINT value is out of range in "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: BIGINT - // UNSIGNED value is out of range in - // '(`database10`.`t0`.`c0` + (`database10`.`t0`.`c0` & (not(0.5))))' - errors.add("Data truncation: BIGINT UNSIGNED value is out of range in "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect - // time value: '0Sly6xqF0' for - // column 'c1' at row 1 - errors.add("Data truncation: Incorrect time value: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: DOUBLE - // value is out of range in - // '(`database0`.`t0`.`c0` * `database0`.`t0`.`c0`)' - errors.add("Data truncation: DOUBLE value is out of range in "); - // java.sql.SQLException: Numeric result of an expression is too large and - // cannot be handled by tianmu. - errors.add("Numeric result of an expression is too large and cannot be handled by tianmu."); - // java.sql.SQLSyntaxErrorException: Unknown column '1020726100' in 'order - // clause' - errors.add("Unknown column "); - if (globalState.getDbmsSpecificOptions().test80Version) { - // Caused by: java.sql.SQLException: Incorrect DATETIME value: '292269055-12-02 - // 16:47:04' - errors.add("Incorrect DATETIME value: "); - // Caused by: java.sql.SQLException: Incorrect TIMESTAMP value: '292269055-12-02 - // 16:47:04' - errors.add("Incorrect TIMESTAMP value: "); - } - - return errors; - } - - public static void addExpectedExpressionErrors(StoneDBGlobalState globalState, ExpectedErrors errors) { - errors.addAll(getExpectedExpressionErrors(globalState)); - } -} diff --git a/src/sqlancer/stonedb/StoneDBOptions.java b/src/sqlancer/stonedb/StoneDBOptions.java deleted file mode 100644 index 0c9d8816a..000000000 --- a/src/sqlancer/stonedb/StoneDBOptions.java +++ /dev/null @@ -1,99 +0,0 @@ -package sqlancer.stonedb; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.stonedb.StoneDBOptions.StoneDBOracleFactory; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.oracle.StoneDBAggregateOracle; -import sqlancer.stonedb.oracle.StoneDBFuzzOracle; -import sqlancer.stonedb.oracle.StoneDBNoRECOracle; -import sqlancer.stonedb.oracle.StoneDBQueryPartitioningDistinctTester; -import sqlancer.stonedb.oracle.StoneDBQueryPartitioningGroupByTester; -import sqlancer.stonedb.oracle.StoneDBQueryPartitioningHavingTester; -import sqlancer.stonedb.oracle.StoneDBQueryPartitioningWhereTester; - -@Parameters(separators = "=", commandDescription = "StoneDB (default host: " + StoneDBOptions.DEFAULT_HOST - + ", default port: " + StoneDBOptions.DEFAULT_PORT + ")") -public class StoneDBOptions implements DBMSSpecificOptions { - public static final String DEFAULT_HOST = "localhost"; - public static final int DEFAULT_PORT = 3306; - - @Parameter(names = "--test-8-version", description = "Let SQLancer test the StoneDB 8.0, otherwise, SQLancer will test the StoneDB 5.7", arity = 1) - public boolean test80Version = true; - - @Parameter(names = "--oracle") - public List oracles = List.of(StoneDBOracleFactory.NOREC); - - public enum StoneDBOracleFactory implements OracleFactory { - FUZZER { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBFuzzOracle(globalState); - } - }, - NOREC { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBNoRECOracle(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new StoneDBQueryPartitioningWhereTester(globalState)); - oracles.add(new StoneDBQueryPartitioningHavingTester(globalState)); - oracles.add(new StoneDBAggregateOracle(globalState)); - oracles.add(new StoneDBQueryPartitioningDistinctTester(globalState)); - oracles.add(new StoneDBQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle<>(oracles, globalState); - } - }, - HAVING { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBAggregateOracle(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(StoneDBGlobalState globalState) throws SQLException { - return new StoneDBQueryPartitioningDistinctTester(globalState); - } - } - } - - @Override - public List getTestOracleFactory() { - return oracles; - } -} diff --git a/src/sqlancer/stonedb/StoneDBProvider.java b/src/sqlancer/stonedb/StoneDBProvider.java deleted file mode 100644 index e4d9c46f8..000000000 --- a/src/sqlancer/stonedb/StoneDBProvider.java +++ /dev/null @@ -1,157 +0,0 @@ -package sqlancer.stonedb; - -import static sqlancer.stonedb.StoneDBBugs.bugNotReported2; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.Statement; - -import com.google.auto.service.AutoService; - -import sqlancer.AbstractAction; -import sqlancer.DatabaseProvider; -import sqlancer.IgnoreMeException; -import sqlancer.MainOptions; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; -import sqlancer.SQLProviderAdapter; -import sqlancer.StatementExecutor; -import sqlancer.common.DBMSCommon; -import sqlancer.common.query.Query; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLQueryProvider; -import sqlancer.stonedb.gen.StoneDBIndexCreateGenerator; -import sqlancer.stonedb.gen.StoneDBIndexDropGenerator; -import sqlancer.stonedb.gen.StoneDBTableAlterGenerator; -import sqlancer.stonedb.gen.StoneDBTableCreateGenerator; -import sqlancer.stonedb.gen.StoneDBTableDeleteGenerator; -import sqlancer.stonedb.gen.StoneDBTableInsertGenerator; -import sqlancer.stonedb.gen.StoneDBTableUpdateGenerator; -import sqlancer.stonedb.gen.StoneDBViewCreateGenerator; - -@AutoService(DatabaseProvider.class) -public class StoneDBProvider extends SQLProviderAdapter { - - public StoneDBProvider() { - super(StoneDBGlobalState.class, StoneDBOptions.class); - } - - public static class StoneDBGlobalState extends SQLGlobalState { - @Override - protected StoneDBSchema readSchema() throws Exception { - return StoneDBSchema.fromConnection(getConnection(), getDatabaseName()); - } - } - - enum Action implements AbstractAction { - TABLES_SHOW((g) -> new SQLQueryAdapter("SHOW TABLES")), // - TABLE_ALTER(StoneDBTableAlterGenerator::generate), // - TABLE_CREATE((g) -> { - String tableName = DBMSCommon.createTableName(g.getSchema().getDatabaseTables().size()); - return StoneDBTableCreateGenerator.generate(g, tableName); - }), // - TABLE_DELETE(StoneDBTableDeleteGenerator::generate), // - TABLE_UPDATE(StoneDBTableUpdateGenerator::generate), // - INDEX_CREATE(StoneDBIndexCreateGenerator::generate), // - INDEX_DROP(StoneDBIndexDropGenerator::generate), // - TABLE_INSERT(StoneDBTableInsertGenerator::generate), // - - VIEW_CREATE((g) -> { - String viewName = g.getSchema().getFreeViewName(); - return StoneDBViewCreateGenerator.generate(g, viewName); - }); - - private final SQLQueryProvider sqlQueryProvider; - - Action(SQLQueryProvider sqlQueryProvider) { - this.sqlQueryProvider = sqlQueryProvider; - } - - @Override - public Query getQuery(StoneDBGlobalState globalState) throws Exception { - return sqlQueryProvider.getQuery(globalState); - } - } - - private static int mapActions(StoneDBGlobalState globalState, Action a) { - Randomly r = globalState.getRandomly(); - switch (a) { - case TABLES_SHOW: - return r.getInteger(0, 1); - case TABLE_ALTER: - return r.getInteger(0, 5); - case TABLE_CREATE: - return r.getInteger(0, 1); - case TABLE_DELETE: - if (bugNotReported2) { - return 0; - } - return r.getInteger(0, 10); - case TABLE_INSERT: - return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - case TABLE_UPDATE: - return r.getInteger(0, 1); - case INDEX_CREATE: - return r.getInteger(0, 1); - case INDEX_DROP: - return r.getInteger(0, 1); - case VIEW_CREATE: - return r.getInteger(0, 1); - default: - throw new AssertionError(a); - } - } - - @Override - public void generateDatabase(StoneDBGlobalState globalState) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < Randomly.smallNumber() + 1) { - String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); - SQLQueryAdapter createTable = StoneDBTableCreateGenerator.generate(globalState, tableName); - globalState.executeStatement(createTable); - } - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - StoneDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public SQLConnection createDatabase(StoneDBGlobalState globalState) throws Exception { - String username = globalState.getOptions().getUserName(); - String password = globalState.getOptions().getPassword(); - String host = globalState.getOptions().getHost(); - int port = globalState.getOptions().getPort(); - if (host == null) { - host = StoneDBOptions.DEFAULT_HOST; - } - if (port == MainOptions.NO_SET_PORT) { - port = StoneDBOptions.DEFAULT_PORT; - } - String databaseName = globalState.getDatabaseName(); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - globalState.getState().logStatement("CREATE DATABASE " + databaseName); - globalState.getState().logStatement("USE " + databaseName); - String url = String.format("jdbc:mysql://%s:%d?serverTimezone=UTC&useSSL=false&allowPublicKeyRetrieval=true", - host, port); - Connection con = DriverManager.getConnection(url, username, password); - try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); - } - try (Statement s = con.createStatement()) { - s.execute("CREATE DATABASE " + databaseName); - } - try (Statement s = con.createStatement()) { - s.execute("USE " + databaseName); - } - return new SQLConnection(con); - } - - @Override - public String getDBMSName() { - return "stonedb"; - } -} diff --git a/src/sqlancer/stonedb/StoneDBSchema.java b/src/sqlancer/stonedb/StoneDBSchema.java deleted file mode 100644 index 346031fba..000000000 --- a/src/sqlancer/stonedb/StoneDBSchema.java +++ /dev/null @@ -1,441 +0,0 @@ -package sqlancer.stonedb; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.schema.AbstractRelationalTable; -import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.common.schema.AbstractTables; -import sqlancer.common.schema.TableIndex; -import sqlancer.stonedb.ast.StoneDBConstant; -import sqlancer.stonedb.ast.StoneDBExpression; - -public class StoneDBSchema extends AbstractSchema { - - public enum StoneDBDataType { - - NULL, TINYINT, SMALLINT, MEDIUMINT, INT, BIGINT, FLOAT, DOUBLE, DECIMAL, YEAR, TIME, DATE, DATETIME, TIMESTAMP, - CHAR, VARCHAR, TINYTEXT, TEXT, MEDIUMTEXT, LONGTEXT, BINARY, VARBINARY, TINYBLOB, BLOB, MEDIUMBLOB, LONGBLOB; - - public static StoneDBDataType getRandomWithoutNull() { - List collect = Stream.of(values()).filter(c -> c != NULL).collect(Collectors.toList()); - return Randomly.fromList(collect); - } - - public static StoneDBDataType getRandom() { - return Randomly.fromOptions(values()); - } - - public static Node getRandomValue(StoneDBDataType dataType, Randomly r) { - switch (dataType) { - case TINYINT: - break; - case SMALLINT: - break; - case MEDIUMINT: - break; - case INT: - return StoneDBConstant.createIntConstant((int) Randomly.getNonCachedInteger()); - case BIGINT: - break; - case FLOAT: - break; - case DOUBLE: - return StoneDBConstant.createDoubleConstant(Randomly.getUncachedDouble()); - case DECIMAL: - break; - case YEAR: - break; - case TIME: - break; - case DATE: - return StoneDBConstant.createDateConstant(Randomly.getNonCachedInteger()); - case DATETIME: - break; - case TIMESTAMP: - return StoneDBConstant.createTimestampConstant(Randomly.getNonCachedInteger()); - case CHAR: - break; - case VARCHAR: - break; - case TINYTEXT: - break; - case TEXT: - return StoneDBConstant.createTextConstant(r.getString()); - case MEDIUMTEXT: - break; - case LONGTEXT: - break; - case BINARY: - break; - case VARBINARY: - break; - case TINYBLOB: - break; - case BLOB: - break; - case MEDIUMBLOB: - break; - case LONGBLOB: - break; - default: - throw new AssertionError(); - } - return null; - } - - public static String getTypeAndValue(StoneDBDataType dataType, Randomly r) { - StringBuilder sb = new StringBuilder(); - switch (dataType) { - case TINYINT: - return "TINYINT"; - case SMALLINT: - return "SMALLINT"; - case MEDIUMINT: - return "MEDIUMINT"; - case INT: - return "INT"; - case BIGINT: - return "BIGINT"; - case FLOAT: - sb.append("FLOAT"); - optionallyAddPrecisionAndScale(sb); - return sb.toString(); - case DOUBLE: - sb.append("DOUBLE"); - optionallyAddPrecisionAndScale(sb); - return sb.toString(); - case DECIMAL: - return "DECIMAL"; // The default value is P(10,0); - case YEAR: - return "YEAR"; - case TIME: - return "TIME"; - case DATE: - return "DATE"; - case DATETIME: - return "DATETIME"; - case TIMESTAMP: - return "TIMESTAMP"; - case CHAR: - sb.append("CHAR").append(Randomly.fromOptions("", "(" + r.getInteger(0, 255) + ")")); - return sb.toString(); - case VARCHAR: - sb.append("VARCHAR").append("(").append(r.getInteger(0, 65535)).append(")"); - return sb.toString(); - case TINYTEXT: - return "TINYTEXT"; - case TEXT: - return "TEXT"; - case MEDIUMTEXT: - return "MEDIUMTEXT"; - case LONGTEXT: - return "LONGTEXT"; - case BINARY: - return "BINARY"; - case VARBINARY: - sb.append("VARBINARY").append("(").append(r.getInteger(0, 65535)).append(")"); - return sb.toString(); - case TINYBLOB: - return "TINYBLOB"; - case BLOB: - return "BLOB"; - case MEDIUMBLOB: - return "MEDIUMBLOB"; - case LONGBLOB: - return "LONGBLOB"; - default: - throw new AssertionError(); - } - } - - private static void optionallyAddPrecisionAndScale(StringBuilder sb) { - if (Randomly.getBoolean()) { - sb.append("("); - // The maximum number of digits (M) for DECIMAL is 65 - long m = Randomly.getNotCachedInteger(1, 65); - sb.append(m); - sb.append(", "); - // The maximum number of supported decimals (D) is 30 - long nCandidate = Randomly.getNotCachedInteger(1, 30); - // For float(M,D), double(M,D) or decimal(M,D), M must be >= D (column 'c0'). - long n = Math.min(nCandidate, m); - sb.append(n); - sb.append(")"); - } - } - } - - public static class StoneDBTable - extends AbstractRelationalTable { - - public StoneDBTable(String tableName, List columns, List indexes, boolean isView) { - super(tableName, columns, indexes, isView); - } - - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - - } - - public static class StoneDBTables extends AbstractTables { - public StoneDBTables(List tables) { - super(tables); - } - } - - public static final class StoneDBIndex extends TableIndex { - private StoneDBIndex(String indexName) { - super(indexName); - } - - public static StoneDBIndex create(String indexName) { - return new StoneDBIndex(indexName); - } - - @Override - public String getIndexName() { - if (super.getIndexName().contentEquals("PRIMARY")) { - return "`PRIMARY`"; - } else { - return super.getIndexName(); - } - } - } - - public static StoneDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { - List databaseTables = new ArrayList<>(); - List tableNames = getTableNames(con, databaseName); - for (String tableName : tableNames) { - List databaseColumns = getTableColumns(con, databaseName, tableName); - List indexes = getIndexes(con, databaseName, tableName); - boolean isView = tableName.startsWith("v"); - StoneDBTable t = new StoneDBTable(tableName, databaseColumns, indexes, isView); - for (StoneDBColumn c : databaseColumns) { - c.setTable(t); - } - databaseTables.add(t); - - } - return new StoneDBSchema(databaseTables); - } - - private static List getIndexes(SQLConnection con, String databaseName, String tableName) - throws SQLException { - List indexes = new ArrayList<>(); - try (ResultSet rs = con.createStatement() - .executeQuery("SELECT INDEX_NAME FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA = '" - + databaseName + "' AND TABLE_NAME='" + tableName + "';")) { - while (rs.next()) { - String indexName = rs.getString("INDEX_NAME"); - indexes.add(StoneDBIndex.create(indexName)); - } - } - return indexes; - } - - private static List getTableColumns(SQLConnection con, String databaseName, String tableName) - throws SQLException { - List columns = new ArrayList<>(); - try (ResultSet rs = con.createStatement() - .executeQuery("select * from information_schema.columns where table_schema = '" + databaseName - + "' AND TABLE_NAME='" + tableName + "';")) { - while (rs.next()) { - String columnName = rs.getString("COLUMN_NAME"); - String dataType = rs.getString("DATA_TYPE"); - int precision = rs.getInt("NUMERIC_PRECISION"); - boolean isNullable = !rs.getString("IS_NULLABLE").equals("NO"); - boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); - StoneDBColumn c = new StoneDBColumn(columnName, getColumnCompositeDataType(dataType), isPrimaryKey, - isNullable, precision); - columns.add(c); - } - } - return columns; - } - - private static StoneDBCompositeDataType getColumnCompositeDataType(String typeString) { - if (Arrays.stream(StoneDBDataType.values()).noneMatch(e -> e.name().equals(typeString.toUpperCase()))) { - throw new AssertionError(typeString); - } - return new StoneDBCompositeDataType(StoneDBDataType.valueOf(typeString.toUpperCase())); - } - - public static StoneDBDataType getColumnDataType(String typeString) { - if (Arrays.stream(StoneDBDataType.values()).noneMatch(e -> e.name().equals(typeString.toUpperCase()))) { - throw new AssertionError(typeString); - } - return StoneDBDataType.valueOf(typeString.toUpperCase()); - } - - private static List getTableNames(SQLConnection con, String databaseName) throws SQLException { - List tableNames = new ArrayList<>(); - try (ResultSet rs = con.createStatement() - .executeQuery("select TABLE_NAME, ENGINE from information_schema.TABLES where table_schema = '" - + databaseName + "';")) { - while (rs.next()) { - if (rs.getString("ENGINE").equals("TIANMU")) { - tableNames.add(rs.getString("TABLE_NAME")); - } - } - } - return tableNames; - } - - public static class StoneDBColumn extends AbstractTableColumn { - - private final boolean isPrimaryKey; - private final boolean isNullable; - private final int precision; - - public StoneDBColumn(String name, StoneDBCompositeDataType columnType, boolean isPrimaryKey, boolean isNullable, - int precision) { - super(name, null, columnType); - this.isPrimaryKey = isPrimaryKey; - this.isNullable = isNullable; - this.precision = precision; - } - - public boolean isPrimaryKey() { - return isPrimaryKey; - } - - public boolean isNullable() { - return isNullable; - } - - public int getPrecision() { - return precision; - } - - } - - public StoneDBSchema(List databaseTables) { - super(databaseTables); - } - - public static class StoneDBCompositeDataType { - private final StoneDBDataType dataType; - private final int size; - - public StoneDBCompositeDataType(StoneDBDataType dataType, int size) { - this.dataType = dataType; - this.size = size; - } - - public StoneDBCompositeDataType(StoneDBDataType dataType) { - this.dataType = dataType; - int size = -1; - switch (dataType) { - case TINYINT: - size = 1; - break; - case SMALLINT: - size = 2; - break; - case MEDIUMINT: - size = 3; - break; - case INT: - size = 4; - break; - case BIGINT: - size = 8; - break; - case FLOAT: - size = 4; - break; - case DOUBLE: - size = 8; - break; - case DECIMAL: - size = -2; - break; - case YEAR: - size = -2; - break; - case TIME: - size = -2; - break; - case DATE: - size = -2; - break; - case DATETIME: - size = -2; - break; - case TIMESTAMP: - size = -2; - break; - case CHAR: - size = -2; - break; - case VARCHAR: - size = -2; - break; - case TINYTEXT: - size = -2; - break; - case TEXT: - size = -2; - break; - case MEDIUMTEXT: - size = -2; - break; - case LONGTEXT: - size = -2; - break; - case BINARY: - size = -2; - break; - case VARBINARY: - size = -2; - break; - case TINYBLOB: - size = -2; - break; - case BLOB: - size = -2; - break; - case MEDIUMBLOB: - size = -2; - break; - case LONGBLOB: - size = -2; - break; - default: - throw new AssertionError(); - } - this.size = size; - } - - public StoneDBDataType getPrimitiveDataType() { - return dataType; - } - - public int getSize() { - if (size == -1) { - throw new AssertionError(this); - } - return size; - } - - public static StoneDBCompositeDataType getRandomWithoutNull() { - StoneDBDataType type = StoneDBDataType.getRandomWithoutNull(); - return new StoneDBCompositeDataType(type); - } - } - - public StoneDBTables getRandomTableNonEmptyTables() { - return new StoneDBTables(Randomly.nonEmptySubset(getDatabaseTables())); - } -} diff --git a/src/sqlancer/stonedb/StoneDBToStringVisitor.java b/src/sqlancer/stonedb/StoneDBToStringVisitor.java deleted file mode 100644 index 971297ddd..000000000 --- a/src/sqlancer/stonedb/StoneDBToStringVisitor.java +++ /dev/null @@ -1,101 +0,0 @@ -package sqlancer.stonedb; - -import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.ast.StoneDBConstant; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; - -public class StoneDBToStringVisitor extends NewToStringVisitor { - @Override - public void visitSpecific(Node expr) { - if (expr instanceof StoneDBConstant) { - visit((StoneDBConstant) expr); - } else if (expr instanceof StoneDBSelect) { - visit((StoneDBSelect) expr); - } else if (expr instanceof StoneDBJoin) { - visit((StoneDBJoin) expr); - } else if (expr instanceof StoneDBCastOperation) { - visit((StoneDBCastOperation) expr); - } else { - throw new AssertionError(expr.getClass()); - } - } - - private void visit(StoneDBConstant constant) { - sb.append(constant.toString()); - } - - private void visit(StoneDBSelect select) { - sb.append("SELECT "); - if (select.isDistinct()) { - sb.append("DISTINCT "); - } - visit(select.getFetchColumns()); - sb.append(" FROM "); - visit(select.getFromList()); - if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { - sb.append(", "); - } - if (!select.getJoinList().isEmpty()) { - visit(select.getJoinList()); - } - if (select.getWhereClause() != null) { - sb.append(" WHERE "); - visit(select.getWhereClause()); - } - if (!select.getGroupByExpressions().isEmpty()) { - sb.append(" GROUP BY "); - visit(select.getGroupByExpressions()); - } - if (select.getHavingClause() != null) { - sb.append(" HAVING "); - visit(select.getHavingClause()); - } - if (!select.getOrderByClauses().isEmpty()) { - sb.append(" ORDER BY "); - visit(select.getOrderByClauses()); - } - if (select.getLimitClause() != null) { - sb.append(" LIMIT "); - visit(select.getLimitClause()); - } - if (select.getOffsetClause() != null) { - sb.append(" OFFSET "); - visit(select.getOffsetClause()); - } - } - - private void visit(StoneDBJoin join) { - visit(join.getLeftTable()); - sb.append(" "); - sb.append(join.getJoinType()); - sb.append(" "); - if (join.getNaturalJoinType() != null) { - sb.append(join.getNaturalJoinType()); - } - sb.append(" JOIN "); - visit(join.getRightTable()); - if (join.getOnCondition() != null) { - sb.append(" ON "); - visit(join.getOnCondition()); - } - } - - private void visit(StoneDBCastOperation cast) { - sb.append("CAST("); - visit(cast.getExpr()); - sb.append(" AS "); - sb.append(cast.getType() == StoneDBDataType.INT ? "UNSIGNED" : cast.getType().toString()); - sb.append(") "); - } - - public static String asString(Node expr) { - StoneDBToStringVisitor visitor = new StoneDBToStringVisitor(); - visitor.visit(expr); - return visitor.get(); - } -} diff --git a/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java b/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java deleted file mode 100644 index 9c49f2678..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBAdvancedFunction.java +++ /dev/null @@ -1,34 +0,0 @@ -package sqlancer.stonedb.ast; - -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.FunctionNode; -import sqlancer.stonedb.ast.StoneDBAdvancedFunction.StoneDBAdvancedFunc; - -public class StoneDBAdvancedFunction extends FunctionNode - implements StoneDBExpression { - - // https://stonedb.io/docs/SQL-reference/functions/advanced-functions - public enum StoneDBAdvancedFunc { - IFNULL(2), IF(3), NULLIF(2), BIN(1), BINARY(1), CONV(3); - - private int nrArgs; - - StoneDBAdvancedFunc(int nrArgs) { - this.nrArgs = nrArgs; - } - - public static StoneDBAdvancedFunc getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - } - - protected StoneDBAdvancedFunction(StoneDBAdvancedFunc function, List args) { - super(function, args); - } -} diff --git a/src/sqlancer/stonedb/ast/StoneDBAggregate.java b/src/sqlancer/stonedb/ast/StoneDBAggregate.java deleted file mode 100644 index f65b54d74..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBAggregate.java +++ /dev/null @@ -1,34 +0,0 @@ -package sqlancer.stonedb.ast; - -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.FunctionNode; -import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; - -public class StoneDBAggregate extends FunctionNode - implements StoneDBExpression { - - // https://stonedb.io/docs/SQL-reference/functions/aggregate-functions/ - public enum StoneDBAggregateFunction { - MAX(1), MIN(1), AVG(1), COUNT(1), SUM(1); - - private int nrArgs; - - StoneDBAggregateFunction(int nrArgs) { - this.nrArgs = nrArgs; - } - - public static StoneDBAggregateFunction getRandom() { - return Randomly.fromOptions(values()); - } - - public int getNrArgs() { - return nrArgs; - } - } - - protected StoneDBAggregate(StoneDBAggregateFunction function, List args) { - super(function, args); - } -} diff --git a/src/sqlancer/stonedb/ast/StoneDBConstant.java b/src/sqlancer/stonedb/ast/StoneDBConstant.java deleted file mode 100644 index 6760bc25c..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBConstant.java +++ /dev/null @@ -1,211 +0,0 @@ -package sqlancer.stonedb.ast; - -import static sqlancer.stonedb.StoneDBBugs.bugNotReported1; - -import java.sql.Timestamp; -import java.text.SimpleDateFormat; - -import sqlancer.common.ast.newast.Node; - -public class StoneDBConstant implements Node { - - private StoneDBConstant() { - } - - public static class StoneDBNullConstant extends StoneDBConstant { - @Override - public String toString() { - return "NULL"; - } - } - - public static Node createNullConstant() { - return new StoneDBNullConstant(); - } - - public static class StoneDBIntConstant extends StoneDBConstant { - private final Integer value; - - public StoneDBIntConstant(int value) { - this.value = value; - } - - @Override - public String toString() { - if (value.equals(Integer.MIN_VALUE)) { - return "INT_NULL"; - } - return String.valueOf(value); - } - } - - public static Node createIntConstant(int val) { - return new StoneDBIntConstant(val); - } - - public static class StoneDBBigIntConstant extends StoneDBConstant { - private final Long value; - - public StoneDBBigIntConstant(long value) { - this.value = value; - } - - @Override - public String toString() { - // For INT type: -2147483648 is reserved to indicate INT_NULL in Tianmu engine, Minimum Value Signed start - // from -2147483647 - // refer: https://stonedb.io/docs/SQL-reference/data-types - if (value.equals(Long.MIN_VALUE)) { - return "BIGINT_NULL"; - } - return String.valueOf(value); - } - } - - public static class StoneDBDoubleConstant extends StoneDBConstant { - - private final Double value; - - public StoneDBDoubleConstant(double value) { - this.value = value; - } - - public double getValue() { - return value; - } - - @Override - public String toString() { - if (value == Double.POSITIVE_INFINITY) { - return "'+Inf'"; - } else if (value == Double.NEGATIVE_INFINITY) { - return "'-Inf'"; - } - return String.valueOf(value); - } - - } - - public static Node createDoubleConstant(double val) { - if (bugNotReported1 && val < 0.5 && val > 0) { - return new StoneDBDoubleConstant(0.5); - } - return new StoneDBDoubleConstant(val); - } - - public static class StoneDBTextConstant extends StoneDBConstant { - - private final String value; - - public StoneDBTextConstant(String value) { - this.value = value; - } - - public String getValue() { - return value; - } - - @Override - public String toString() { - return "'" + value.replace("'", "''") + "'"; - } - - } - - public static Node createTextConstant(String text) { - return new StoneDBTextConstant(text); - } - - public static class StoneDBBitConstant extends StoneDBConstant { - - private final String value; - - public StoneDBBitConstant(long value) { - this.value = Long.toBinaryString(value); - } - - public String getValue() { - return value; - } - - @Override - public String toString() { - return "B'" + value + "'"; - } - - } - - public static class StoneDBDateConstant extends StoneDBConstant { - - public String textRepr; - - public StoneDBDateConstant(long val) { - Timestamp timestamp = new Timestamp(val); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); - textRepr = dateFormat.format(timestamp); - } - - public String getValue() { - return textRepr; - } - - @Override - public String toString() { - return String.format("DATE '%s'", textRepr); - } - - } - - public static Node createDateConstant(long integer) { - return new StoneDBDateConstant(integer); - } - - public static class StoneDBTimestampConstant extends StoneDBConstant { - - public String textRepr; - - public StoneDBTimestampConstant(long val) { - Timestamp timestamp = new Timestamp(val); - SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - textRepr = dateFormat.format(timestamp); - } - - public String getValue() { - return textRepr; - } - - @Override - public String toString() { - return String.format("TIMESTAMP '%s'", textRepr); - } - - } - - public static Node createTimestampConstant(long integer) { - return new StoneDBTimestampConstant(integer); - } - - public static class StoneDBBooleanConstant extends StoneDBConstant { - - private final boolean value; - - public StoneDBBooleanConstant(boolean value) { - this.value = value; - } - - public boolean getValue() { - return value; - } - - @Override - public String toString() { - return String.valueOf(value); - } - - } - - public static Node createBooleanConstant(boolean val) { - return new StoneDBBooleanConstant(val); - } - -} diff --git a/src/sqlancer/stonedb/ast/StoneDBExpression.java b/src/sqlancer/stonedb/ast/StoneDBExpression.java deleted file mode 100644 index fb8e75124..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBExpression.java +++ /dev/null @@ -1,4 +0,0 @@ -package sqlancer.stonedb.ast; - -public interface StoneDBExpression { -} diff --git a/src/sqlancer/stonedb/ast/StoneDBJoin.java b/src/sqlancer/stonedb/ast/StoneDBJoin.java deleted file mode 100644 index 8bfff08ee..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBJoin.java +++ /dev/null @@ -1,126 +0,0 @@ -package sqlancer.stonedb.ast; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator; - -public class StoneDBJoin implements Node { - - public enum JoinType { - INNER, NATURAL, LEFT, RIGHT; - - public static JoinType getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum NaturalJoinType { - LEFT, RIGHT; - - public static NaturalJoinType getRandom() { - return Randomly.fromOptions(values()); - } - } - - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; - private final JoinType joinType; - private final Node onCondition; - private NaturalJoinType naturalJoinType; - - public StoneDBJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node onCondition) { - this.leftTable = leftTable; - this.rightTable = rightTable; - this.joinType = joinType; - this.onCondition = onCondition; - } - - public TableReferenceNode getLeftTable() { - return leftTable; - } - - public TableReferenceNode getRightTable() { - return rightTable; - } - - public JoinType getJoinType() { - return joinType; - } - - public Node getOnCondition() { - return onCondition; - } - - public NaturalJoinType getNaturalJoinType() { - return naturalJoinType; - } - - public void setNaturalJoinType(NaturalJoinType naturalJoinType) { - this.naturalJoinType = naturalJoinType; - } - - public static List> getJoins( - List> tableList, StoneDBGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); - while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - // get two tables to join - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); - // store all columns in the above two tables - List columns = new ArrayList<>(leftTable.getTable().getColumns()); - columns.addAll(rightTable.getTable().getColumns()); - // create a join generator - StoneDBExpressionGenerator joinGen = new StoneDBExpressionGenerator(globalState).setColumns(columns); - switch (StoneDBJoin.JoinType.getRandom()) { - case INNER: - joinExpressions.add(StoneDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); - break; - case NATURAL: - joinExpressions.add(StoneDBJoin.createNaturalJoin(leftTable, rightTable, NaturalJoinType.getRandom())); - break; - case LEFT: - joinExpressions - .add(StoneDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); - break; - case RIGHT: - joinExpressions - .add(StoneDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); - break; - default: - throw new AssertionError(); - } - } - return joinExpressions; - } - - public static StoneDBJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node onClause) { - return new StoneDBJoin(left, right, JoinType.RIGHT, onClause); - } - - public static StoneDBJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node onClause) { - return new StoneDBJoin(left, right, JoinType.LEFT, onClause); - } - - public static StoneDBJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node onClause) { - return new StoneDBJoin(left, right, JoinType.INNER, onClause); - } - - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, NaturalJoinType naturalJoinType) { - StoneDBJoin join = new StoneDBJoin(left, right, JoinType.NATURAL, null); - join.setNaturalJoinType(naturalJoinType); - return join; - } -} diff --git a/src/sqlancer/stonedb/ast/StoneDBSelect.java b/src/sqlancer/stonedb/ast/StoneDBSelect.java deleted file mode 100644 index 081106260..000000000 --- a/src/sqlancer/stonedb/ast/StoneDBSelect.java +++ /dev/null @@ -1,17 +0,0 @@ -package sqlancer.stonedb.ast; - -import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; - -public class StoneDBSelect extends SelectBase> implements Node { - - private boolean isDistinct; - - public void setDistinct(boolean isDistinct) { - this.isDistinct = isDistinct; - } - - public boolean isDistinct() { - return isDistinct; - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java b/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java deleted file mode 100644 index 3e1dcd348..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBExpressionGenerator.java +++ /dev/null @@ -1,340 +0,0 @@ -package sqlancer.stonedb.gen; - -import static sqlancer.stonedb.StoneDBBugs.bug1942; -import static sqlancer.stonedb.StoneDBBugs.bugNotReported3; -import static sqlancer.stonedb.StoneDBBugs.bugNotReported6; -import static sqlancer.stonedb.StoneDBBugs.bugNotReportedXOR; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewCaseOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.ast.StoneDBAdvancedFunction.StoneDBAdvancedFunc; -import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; -import sqlancer.stonedb.ast.StoneDBConstant; -import sqlancer.stonedb.ast.StoneDBExpression; - -public class StoneDBExpressionGenerator extends UntypedExpressionGenerator, StoneDBColumn> { - - private final StoneDBGlobalState globalState; - - public StoneDBExpressionGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - } - - private enum Expression { - UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, BINARY_BITWISE, BETWEEN, IN, - NOT_IN, CASE - } - - public static class StoneDBCastOperation implements Node { - Node expr; - StoneDBDataType type; - - public StoneDBCastOperation(Node expr, StoneDBDataType type) { - this.expr = expr; - this.type = type; - } - - public Node getExpr() { - return expr; - } - - public StoneDBDataType getType() { - return type; - } - } - - @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, StoneDBUnaryPrefixOperator.NOT); - } - - @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, StoneDBUnaryPostfixOperator.IS_NULL); - } - - @Override - public Node generateConstant() { - if (Randomly.getBooleanWithSmallProbability()) { - return StoneDBConstant.createNullConstant(); - } - StoneDBDataType type = StoneDBDataType.getRandomWithoutNull(); - return generateConstant(type); - } - - public Node generateConstant(StoneDBDataType dataType) { - switch (dataType) { - case INT: - return StoneDBConstant - .createIntConstant(globalState.getRandomly().getInteger(Integer.MIN_VALUE + 1, Integer.MAX_VALUE)); - case DATE: - return StoneDBConstant.createDateConstant(globalState.getRandomly().getInteger()); - case TIMESTAMP: - return StoneDBConstant.createTimestampConstant(globalState.getRandomly().getInteger()); - case VARCHAR: - return StoneDBConstant.createTextConstant(globalState.getRandomly().getString()); - case DOUBLE: - return StoneDBConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - default: - throw new IgnoreMeException(); - } - } - - public Node generateConstant(StoneDBDataType dataType, boolean isNullable) { - if (isNullable && Randomly.getBooleanWithSmallProbability()) { - generateConstant(StoneDBDataType.NULL); - } - return generateConstant(dataType); - } - - @Override - protected Node generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - if (allowAggregates && Randomly.getBoolean()) { - StoneDBAggregateFunction aggregateFunction = StoneDBAggregateFunction.getRandom(); - allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs(), depth + 1), - aggregateFunction); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - StoneDBAdvancedFunc advancedFunction = StoneDBAdvancedFunc.getRandom(); - return new NewFunctionNode<>(generateExpressions(advancedFunction.getNrArgs(), depth + 1), - advancedFunction); - } - List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - Expression expr = Randomly.fromList(possibleOptions); - Operator op; - switch (expr) { - case UNARY_PREFIX: - op = StoneDBUnaryPrefixOperator.getRandom(); - return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), op); - case UNARY_POSTFIX: - if (!bug1942) { - op = StoneDBUnaryPostfixOperator.getRandom(); - } else if (!bugNotReported3) { - op = StoneDBUnaryPostfixOperator.IS_NULL; - } else { - throw new IgnoreMeException(); - } - return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), op); - case BINARY_COMPARISON: - if (!bugNotReported6) { - op = StoneDBBinaryComparisonOperator.getRandom(); - } else { - throw new IgnoreMeException(); - } - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case IN: - return new NewInOperatorNode<>(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), false); - case NOT_IN: - return new NewInOperatorNode<>(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), true); - case BINARY_LOGICAL: - if (bugNotReportedXOR) { - op = Randomly.fromList(Arrays.stream(StoneDBBinaryLogicalOperator.values()) - .filter(p -> p != StoneDBBinaryLogicalOperator.XOR).collect(Collectors.toList())); - } else { - op = StoneDBBinaryLogicalOperator.getRandom(); - } - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case BINARY_ARITHMETIC: - op = StoneDBBinaryArithmeticOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case BINARY_BITWISE: - op = StoneDBBinaryBitwiseOperator.getRandom(); - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), op); - case BETWEEN: - return new NewBetweenOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), - generateExpression(depth + 1), Randomly.getBoolean()); - case CASE: - int nr = Randomly.smallNumber() + 1; - return new NewCaseOperatorNode<>(generateExpression(depth + 1), generateExpressions(nr, depth + 1), - generateExpressions(nr, depth + 1), generateExpression(depth + 1)); - default: - throw new AssertionError(); - } - } - - @Override - protected Node generateColumn() { - StoneDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode<>(column); - } - - protected List> generateColumns() { - int size = globalState.getRandomly().getInteger(1, columns.size()); - Set> set = new HashSet<>(); - while (set.size() < size) { - set.add(generateColumn()); - } - return new ArrayList<>(set); - } - - public enum StoneDBUnaryPrefixOperator implements Operator { - - NOT("NOT"), PLUS("+"), MINUS("-"), INVERSION("!"); - - private final String textRepr; - - StoneDBUnaryPrefixOperator(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - public static StoneDBUnaryPrefixOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - public enum StoneDBUnaryPostfixOperator implements Operator { - - IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); - - private final String textRepr; - - StoneDBUnaryPostfixOperator(String textRepr) { - this.textRepr = textRepr; - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - public static StoneDBUnaryPostfixOperator getRandom() { - return Randomly.fromOptions(values()); - } - } - - /* - * Comparison operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/comparison-operators - */ - public enum StoneDBBinaryComparisonOperator implements Operator { - EQUAL("="), GREATER(">"), LESS("<"), GREATER_EQUAL(">="), LESS_EQUAL("<="), - NOT_EQUALS(Randomly.fromList(Arrays.asList("!=", "<>"))), NULL_SAFE_EQUAL("<=>"), LIKE("LIKE"); - - private final String textRepr; - - StoneDBBinaryComparisonOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - } - - /* - * Logical operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/logical-operators - */ - public enum StoneDBBinaryLogicalOperator implements Operator { - - AND("AND"), OR("OR"), XOR("XOR"); - - private final String textRepr; - - StoneDBBinaryLogicalOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - /* - * Arithmetic operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/arithmetic-operators - */ - public enum StoneDBBinaryArithmeticOperator implements Operator { - ADDITION("+"), MINUS("-"), MULTIPLICATION("*"), DIVISION(Randomly.fromList(Arrays.asList("/", " div "))), - MODULO(Randomly.fromList(Arrays.asList("%", " mod "))); - - private final String textRepr; - - StoneDBBinaryArithmeticOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - /* - * Bitwise operators supported by StoneDB: https://stonedb.io/docs/SQL-reference/operators/bitwise-operators - */ - public enum StoneDBBinaryBitwiseOperator implements Operator { - AND("&"), OR("|"), XOR("^"), LEFTSHIFT("<<"), RIGHTSHIFT(">>"); - - private final String textRepr; - - StoneDBBinaryBitwiseOperator(String textRepr) { - this.textRepr = textRepr; - } - - public static Operator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return textRepr; - } - - } - - public NewFunctionNode generateAggregateAndArgs( - StoneDBAggregateFunction aggregateFunction) { - return new NewFunctionNode<>(generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); - } - - public Node generateAggregate() { - StoneDBAggregateFunction aggrFunc = StoneDBAggregateFunction.getRandom(); - return generateAggregateAndArgs(aggrFunc); - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java deleted file mode 100644 index 82eab3413..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBIndexCreateGenerator.java +++ /dev/null @@ -1,121 +0,0 @@ -package sqlancer.stonedb.gen; - -import sqlancer.Randomly; -import sqlancer.Randomly.StringGenerationStrategy; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; - -public class StoneDBIndexCreateGenerator { - private final StoneDBGlobalState globalState; - private final Randomly r; - // which table to add index - StoneDBTable table; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBIndexCreateGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - r = globalState.getRandomly(); - table = globalState.getSchema().getRandomTable(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBIndexCreateGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("CREATE "); - // Tianmu engine does not support fulltext index. - sb.append(Randomly.fromOptions("UNIQUE" /* "FULLTEXT" */, "SPATIAL")); - sb.append(" INDEX "); - sb.append(globalState.getSchema().getFreeIndexName()); - if (Randomly.getBoolean()) { - appendIndexType(); - } - sb.append(" ON "); - sb.append(table.getName()); - appendKeyParts(); - appendIndexOption(); - appendAlgoOrLockOption(); - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void addExpectedErrors() { - // java.sql.SQLException: Tianmu engine does not support unique index. - errors.add("Tianmu engine does not support unique index"); - // java.sql.SQLException: Tianmu engine does not support fulltext index. - errors.add("Tianmu engine does not support fulltext index"); - // java.sql.SQLException: The used table type doesn't support SPATIAL indexes - errors.add("The used table type doesn't support SPATIAL indexes"); - // java.sql.SQLException: ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY. - errors.add("ALGORITHM=INPLACE is not supported for this operation. Try ALGORITHM=COPY."); - // java.sql.SQLSyntaxErrorException: Key column 'c0' doesn't exist in table - errors.add("doesn't exist in table"); - // java.sql.SQLSyntaxErrorException: A SPATIAL index may only contain a geometrical type column - errors.add("A SPATIAL index may only contain a geometrical type column"); - } - - private void appendIndexType() { - // The index_type clause cannot be used for FULLTEXT INDEX or SPATIAL INDEX specifications. - if (sb.toString().contains("FULLTEXT") || sb.toString().contains("SPATIAL")) { - return; - } - if (Randomly.getBoolean()) { - return; - } - sb.append(" USING "); - sb.append(Randomly.fromOptions("BTREE", "HASH")); - } - - private void appendKeyParts() { - int numberOfKeyParts = Randomly.fromOptions(1, 1, 1, 1, table.getColumns().size()); - for (int i = 0; i < numberOfKeyParts; i++) { - appendKeyPart(); - } - } - - private void appendKeyPart() { - sb.append("("); - StoneDBColumn randomColumn = table.getRandomColumn(); - sb.append(randomColumn.getName()); - if (Randomly.getBoolean()) { - sb.append(" (").append(Randomly.smallNumber()).append(")"); - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions(" ASC", " DESC")); - } - sb.append(")"); - } - - private void appendIndexOption() { - if (Randomly.getBoolean()) { - return; - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions("KEY_BLOCK_SIZE ", "KEY_BLOCK_SIZE = ")); - sb.append(r.getInteger(1, Math.max(1, Randomly.smallNumber()))); - sb.append(" "); - } - if (Randomly.getBoolean()) { - StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; - sb.append(String.format("COMMENT '%s' ", strategy.getString(r))); - } - } - - private void appendAlgoOrLockOption() { - if (Randomly.getBoolean()) { - return; - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); - sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); - } else { - sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); - sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java b/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java deleted file mode 100644 index dda72c735..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBIndexDropGenerator.java +++ /dev/null @@ -1,47 +0,0 @@ -package sqlancer.stonedb.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; - -public class StoneDBIndexDropGenerator { - // which table to drop index - StoneDBSchema.StoneDBTable table; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBIndexDropGenerator(StoneDBGlobalState globalState) { - table = globalState.getSchema().getRandomTable(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBIndexDropGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - if (!table.hasIndexes()) { - return null; - } - sb.append("DROP INDEX "); - sb.append(table.getRandomIndex().getIndexName()); - sb.append(" ON "); - sb.append(table.getName()); - appendAlgoOrLockOption(); - return new SQLQueryAdapter(sb.toString(), errors); - } - - private void appendAlgoOrLockOption() { - if (Randomly.getBoolean()) { - return; - } - if (Randomly.getBoolean()) { - sb.append(Randomly.fromOptions(" ALGORITHM ", " ALGORITHM = ")); - sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY")); - } else { - sb.append(Randomly.fromOptions(" LOCK ", " LOCK = ")); - sb.append(Randomly.fromOptions("DEFAULT", "NONE", "SHARED", "EXCLUSIVE")); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java deleted file mode 100644 index c37024669..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableAlterGenerator.java +++ /dev/null @@ -1,141 +0,0 @@ -package sqlancer.stonedb.gen; - -import java.util.regex.Pattern; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; - -public class StoneDBTableAlterGenerator { - private final StoneDBGlobalState globalState; - private final StringBuilder sb = new StringBuilder(); - private final StoneDBTable table; - ExpectedErrors errors = new ExpectedErrors(); - - enum Action { - ADD_COLUMN, ALTER_COLUMN, DROP_COLUMN, CHANGE_COLUMN - } - - public StoneDBTableAlterGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - table = globalState.getSchema().getRandomTable(t -> !t.isView()); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBTableAlterGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("ALTER TABLE "); - sb.append(table.getName()); - sb.append(" "); - appendAlterOption(Randomly.fromOptions(Action.values())); - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void addExpectedErrors() { - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Data too long for column 'c0' at row 2 - errors.addRegex(Pattern.compile("Data truncation: Data too long for column 'c\\d{1,3}' at row \\d{1,3}")); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: - // '0.571272522740968' for column 'c1' at row 1 - errors.add("Incorrect datetime value: "); - // java.sql.SQLSyntaxErrorException: Invalid default value for 'c0' - errors.add("Invalid default value for "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row - // 2 - errors.add("Data truncation: Out of range value for column "); - // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes - errors.add("Specified key was too long; max key length is 3072 bytes"); - // java.sql.SQLSyntaxErrorException: You can't delete all columns with ALTER TABLE; use DROP TABLE instead - errors.add("You can't delete all columns with ALTER TABLE; use DROP TABLE instead"); - // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 't1' - errors.addRegex(Pattern.compile("Unknown column 'c\\d{1,3}' in 't\\d{1,3}'")); - // java.sql.SQLSyntaxErrorException: BLOB, TEXT, GEOMETRY or JSON column 'c0' can't have a default value - errors.addRegex(Pattern.compile("BLOB, TEXT, GEOMETRY or JSON column 'c\\d{1,3}' can't have a default value")); - // java.sql.SQLSyntaxErrorException: Column length too big for column 'c91' (max = 16383); use BLOB or TEXT - // instead - errors.addRegex(Pattern - .compile("Column length too big for column 'c\\d{1,3}' \\(max = 16383\\); use BLOB or TEXT instead")); - } - - private void appendAlterOption(Action action) { - StoneDBExpressionGenerator generator = new StoneDBExpressionGenerator(globalState) - .setColumns(table.getColumns()); - if (globalState.getDbmsSpecificOptions().test80Version && Randomly.getBooleanWithSmallProbability()) { - sb.append("RENAME COLUMN "); - sb.append(table.getRandomColumn().getName()); - sb.append(" TO "); - sb.append(table.getFreeColumnName()); - } - switch (action) { - case ADD_COLUMN: - sb.append("ADD COLUMN "); - String columnName = table.getFreeColumnName(); - sb.append(" ").append(columnName).append(" "); - sb.append( - StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); - // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT - // instead - errors.addRegex(Pattern - .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); - if (Randomly.getBoolean()) { - if (Randomly.getBoolean()) { - sb.append(" FIRST"); - } else { - sb.append(" AFTER "); - sb.append(table.getRandomColumn().getName()); - } - } - break; - case DROP_COLUMN: - sb.append(Randomly.fromOptions("DROP COLUMN ", "DROP ")); - sb.append(table.getRandomColumn().getName()); - break; - case ALTER_COLUMN: - sb.append(Randomly.fromOptions("ALTER COLUMN ", "ALTER ")); - StoneDBColumn randomColumn = table.getRandomColumn(); - sb.append(randomColumn.getName()); - if (Randomly.getBoolean()) { - sb.append(" SET DEFAULT ").append(generator - .generateConstant(randomColumn.getType().getPrimitiveDataType(), Randomly.getBoolean())); - } else { - sb.append(" DROP DEFAULT"); - } - break; - case CHANGE_COLUMN: - sb.append(Randomly.fromOptions("CHANGE COLUMN ", "CHANGE ")); - String oldColumnName = table.getRandomColumn().getName(); - String newColumnName = table.getFreeColumnName(); - sb.append(oldColumnName).append(" ").append(newColumnName).append(" "); - errors.add("Incorrect integer value: "); - // java.sql.SQLException: Data truncated for column 'c1' at row 1 - errors.add("Data truncated for column "); - sb.append( - StoneDBDataType.getTypeAndValue(StoneDBDataType.getRandomWithoutNull(), globalState.getRandomly())); - // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT - // instead - errors.addRegex(Pattern - .compile("Column length too big for column 'c\\d{1,3}' (max = 16383); use BLOB or TEXT instead")); - // java.sql.SQLSyntaxErrorException: BLOB column 'c1' can't be used in key specification with the used table - // type - errors.addRegex(Pattern - .compile("BLOB column 'c\\d{1,3}' can't be used in key specification with the used table type")); - if (Randomly.getBoolean()) { - if (Randomly.getBoolean()) { - sb.append(" FIRST"); - } else { - sb.append(" AFTER "); - sb.append(table.getRandomColumn().getName()); - } - } - break; - default: - throw new AssertionError(action); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java deleted file mode 100644 index c288c57fd..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableCreateGenerator.java +++ /dev/null @@ -1,266 +0,0 @@ -package sqlancer.stonedb.gen; - -import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.PRIMARY_KEY; -import static sqlancer.stonedb.gen.StoneDBTableCreateGenerator.ColumnOptions.UNIQUE_KEY; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.regex.Pattern; - -import sqlancer.Randomly; -import sqlancer.Randomly.StringGenerationStrategy; -import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; - -public class StoneDBTableCreateGenerator { - // the name of the table to create - private final String tableName; - private final StoneDBSchema schema; - // the name of the columns in the table - private final List columns = new ArrayList<>(); - private final boolean allowPrimaryKey; - private boolean setPrimaryKey; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - private final Randomly r; - - public StoneDBTableCreateGenerator(StoneDBGlobalState globalState, String tableName) { - this.tableName = tableName; - this.schema = globalState.getSchema(); - allowPrimaryKey = Randomly.getBoolean(); - this.r = globalState.getRandomly(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String tableName) { - return new StoneDBTableCreateGenerator(globalState, tableName).getQuery(); - } - - public SQLQueryAdapter getQuery() { - sb.append(Randomly.fromOptions("CREATE TABLE "/* , "CREATE TEMPORARY TABLE " */)); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(tableName); - // ues link statement - if (Randomly.getBoolean() && !schema.getDatabaseTables().isEmpty()) { - sb.append(" LIKE "); - sb.append(schema.getRandomTable().getName()); - } else { - appendColumns(); - if (Randomly.getBoolean()) { - sb.append(" "); - appendTableOptions(); - } - } - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors, true); - } - - private void addExpectedErrors() { - // java.sql.SQLSyntaxErrorException: BLOB/TEXT column 'c0' used in key specification without a key length - errors.add("used in key specification without a key length"); - // java.sql.SQLException: Tianmu engine does not support unique index. - errors.add("Tianmu engine does not support unique index"); - // java.sql.SQLException: BLOB column 'c0' can't be used in key specification with the used table type - errors.add("can't be used in key specification with the used table type"); - // java.sql.SQLSyntaxErrorException: Specified key was too long; max key length is 3072 bytes - errors.add("Specified key was too long; max key length is 3072 bytes"); - // java.sql.SQLSyntaxErrorException: Column length too big for column 'c1' (max = 16383); use BLOB or TEXT - // instead - errors.add("Column length too big for column"); - // BLOB/TEXT column 'c0' used in key specification without a key length - errors.addRegex(Pattern.compile("BLOB/TEXT column 'c.*' used in key specification without a key length")); - } - - private enum TableOptions { - AUTO_INCREMENT, AVG_ROW_LENGTH, CHECKSUM, COMPRESSION, DELAY_KEY_WRITE, /* ENCRYPTION, */ INSERT_METHOD, - KEY_BLOCK_SIZE, MAX_ROWS, MIN_ROWS, PACK_KEYS, STATS_AUTO_RECALC, STATS_PERSISTENT, STATS_SAMPLE_PAGES; - - public static List getRandomTableOptions() { - List options; - // try to ensure that usually, only a few of these options are generated - if (Randomly.getBooleanWithSmallProbability()) { - options = Randomly.subset(TableOptions.values()); - } else { - if (Randomly.getBoolean()) { - options = Collections.emptyList(); - } else { - options = Randomly.nonEmptySubset(Arrays.asList(TableOptions.values()), Randomly.smallNumber()); - } - } - return options; - } - } - - private void appendTableOptions() { - List tableOptions = TableOptions.getRandomTableOptions(); - int i = 0; - for (TableOptions o : tableOptions) { - if (i++ != 0) { - sb.append(", "); - } - switch (o) { - case AUTO_INCREMENT: - sb.append("AUTO_INCREMENT = "); - sb.append(r.getPositiveInteger()); - break; - // The valid range for avg_row_length is [0,4294967295] - case AVG_ROW_LENGTH: - sb.append("AVG_ROW_LENGTH = "); - sb.append(r.getLong(0, 4294967295L + 1)); - break; - case CHECKSUM: - sb.append("CHECKSUM = 1"); - break; - case COMPRESSION: - sb.append("COMPRESSION = '"); - sb.append(Randomly.fromOptions("ZLIB", "LZ4", "NONE")); - sb.append("'"); - break; - case DELAY_KEY_WRITE: - sb.append("DELAY_KEY_WRITE = "); - sb.append(Randomly.fromOptions(0, 1)); - break; - case INSERT_METHOD: - sb.append("INSERT_METHOD = "); - sb.append(Randomly.fromOptions("NO", "FIRST", "LAST")); - break; - // The valid range for key_block_size is [0,65535] - case KEY_BLOCK_SIZE: - sb.append("KEY_BLOCK_SIZE = "); - sb.append(r.getInteger(0, 65535 + 1)); - break; - case MAX_ROWS: - sb.append("MAX_ROWS = "); - sb.append(r.getLong(0, Long.MAX_VALUE)); - break; - case MIN_ROWS: - sb.append("MIN_ROWS = "); - sb.append(r.getLong(1, Long.MAX_VALUE)); - break; - case PACK_KEYS: - sb.append("PACK_KEYS = "); - sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); - break; - case STATS_AUTO_RECALC: - sb.append("STATS_AUTO_RECALC = "); - sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); - break; - case STATS_PERSISTENT: - sb.append("STATS_PERSISTENT = "); - sb.append(Randomly.fromOptions("1", "0", "DEFAULT")); - break; - case STATS_SAMPLE_PAGES: - sb.append("STATS_SAMPLE_PAGES = "); - sb.append(r.getInteger(1, Short.MAX_VALUE)); - break; - default: - throw new AssertionError(o); - } - } - } - - private void appendColumns() { - sb.append("("); - for (int i = 0; i < 1 + Randomly.smallNumber(); i++) { - if (i != 0) { - sb.append(", "); - } - appendColumn(i); - } - sb.append(")"); - } - - private void appendColumn(int columnId) { - String columnName = DBMSCommon.createColumnName(columnId); - columns.add(columnName); - sb.append(columnName); - appendColumnDefinition(); - } - - private void appendColumnDefinition() { - sb.append(" "); - StoneDBDataType randomType = StoneDBDataType.getRandomWithoutNull(); - appendType(randomType); - appendColumnOption(randomType); - } - - protected enum ColumnOptions { - NULL_OR_NOT_NULL, PRIMARY_KEY, UNIQUE_KEY, COMMENT, COLUMN_FORMAT, STORAGE - } - - private void appendColumnOption(StoneDBDataType type) { - boolean isTextType = type == StoneDBDataType.VARCHAR; - boolean isNull = false; - boolean columnHasPrimaryKey = false; - List columnOptions = Randomly.subset(ColumnOptions.values()); - // boolean tableHasNullableColumn; - // if (!columnOptions.contains(ColumnOptions.NULL_OR_NOT_NULL)) { - // tableHasNullableColumn = true; - // } - // only use one key, unique key or primary key, but not both - if (columnOptions.contains(PRIMARY_KEY) && columnOptions.contains(UNIQUE_KEY)) { - columnOptions.remove(Randomly.fromOptions(PRIMARY_KEY, UNIQUE_KEY)); - } - if (isTextType) { - // TODO: restriction due to the limited key length - columnOptions.remove(PRIMARY_KEY); - columnOptions.remove(UNIQUE_KEY); - } - for (ColumnOptions o : columnOptions) { - sb.append(" "); - switch (o) { - case NULL_OR_NOT_NULL: - // PRIMARY KEYs cannot be NULL - if (!columnHasPrimaryKey) { - if (Randomly.getBoolean()) { - sb.append("NULL"); - } - // tableHasNullableColumn = true; - isNull = true; - } else { - sb.append("NOT NULL"); - } - break; - case UNIQUE_KEY: - sb.append("UNIQUE"); - if (Randomly.getBoolean()) { - sb.append(" KEY"); - } - break; - case COMMENT: - StringGenerationStrategy strategy = Randomly.StringGenerationStrategy.ALPHANUMERIC; - sb.append(String.format("COMMENT '%s' ", strategy.getString(r))); - break; - case COLUMN_FORMAT: - sb.append("COLUMN_FORMAT "); - sb.append(Randomly.fromOptions("FIXED", "DYNAMIC", "DEFAULT")); - break; - case STORAGE: - sb.append("STORAGE "); - sb.append(Randomly.fromOptions("DISK", "MEMORY")); - break; - case PRIMARY_KEY: - // PRIMARY KEYs cannot be NULL - if (allowPrimaryKey && !setPrimaryKey && !isNull) { - sb.append("PRIMARY KEY"); - setPrimaryKey = true; - columnHasPrimaryKey = true; - } - break; - default: - throw new AssertionError(); - } - } - } - - private void appendType(StoneDBDataType randomType) { - sb.append(StoneDBDataType.getTypeAndValue(randomType, r)); - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java deleted file mode 100644 index 0d8446d4b..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableDeleteGenerator.java +++ /dev/null @@ -1,70 +0,0 @@ -package sqlancer.stonedb.gen; - -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.stonedb.StoneDBBugs; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public final class StoneDBTableDeleteGenerator { - private final StoneDBGlobalState globalState; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - Randomly r; - - private StoneDBTableDeleteGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - r = globalState.getRandomly(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBTableDeleteGenerator(globalState).getQuery(); - } - - public SQLQueryAdapter getQuery() { - StoneDBTable randomTable = globalState.getSchema().getRandomTable(); - sb.append("DELETE"); - if (Randomly.getBoolean()) { - sb.append(" LOW_PRIORITY"); - } - if (Randomly.getBoolean()) { - sb.append(" QUICK"); - } - if (Randomly.getBoolean()) { - sb.append(" IGNORE"); - } - sb.append(" FROM "); - sb.append(randomTable.getName()); - if (globalState.getDbmsSpecificOptions().test80Version) { - sb.append(" AS "); - sb.append(globalState.getSchema().getFreeTableName()); - } - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) - .setColumns(randomTable.getColumns()).generateExpression())); - } - if (!StoneDBBugs.bug1933 && Randomly.getBoolean()) { - sb.append(" ORDER BY "); - sb.append(String.join(", ", Randomly.fromOptions( - randomTable.getColumns().stream().map(AbstractTableColumn::getName).collect(Collectors.toList()))) - .replace('[', '(').replace(']', ')')); - } - if (Randomly.getBoolean()) { - sb.append(" LIMIT "); - sb.append(r.getInteger(0, (int) randomTable.getNrRows(globalState))); - } - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors); - } - - private void addExpectedErrors() { - // java.sql.SQLException: Incorrect string value: '\xBC\xE7\xC9\x91\x05R...' for column 'c1' at row 1 - errors.add("Incorrect string value: "); - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java deleted file mode 100644 index 0a70c8467..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableInsertGenerator.java +++ /dev/null @@ -1,142 +0,0 @@ -package sqlancer.stonedb.gen; - -import static sqlancer.stonedb.StoneDBBugs.bugNotReported4; -import static sqlancer.stonedb.StoneDBBugs.bugNotReported5; - -import java.util.List; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBTableInsertGenerator extends AbstractInsertGenerator { - private final StoneDBGlobalState globalState; - // which table to insert into - private final StoneDBTable table; - // which subset columns of the table to add values - private final List columns; - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBTableInsertGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - table = globalState.getSchema().getRandomTable(); - columns = table.getRandomNonEmptyColumnSubset(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBTableInsertGenerator(globalState).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("INSERT"); - if (Randomly.getBoolean()) { - sb.append(" "); - sb.append(Randomly.fromOptions("LOW_PRIORITY", "DELAYED", "HIGH_PRIORITY")); - } - if (!bugNotReported4 && Randomly.getBoolean()) { - sb.append(" IGNORE"); - } - sb.append(" INTO "); - sb.append(table.getName()); - appendPartition(); - appendColumnsAndValues(columns); - appendAS(); - appendOnDuplicateUpdate(); - addExpectedErrors(); - return new SQLQueryAdapter(sb.toString(), errors); - } - - private void addExpectedErrors() { - // java.sql.SQLException: Incorrect DATE value: '292278994-08-17' - errors.add("Incorrect DATE value: '"); - // java.sql.SQLIntegrityConstraintViolationException: Duplicate entry '1970-01-14' for key 'PRIMARY' - errors.add("Duplicate entry "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Out of range value for column 'c0' at row - errors.add("Data truncation: Out of range value for column "); - // java.sql.SQLSyntaxErrorException: Unknown column 'c0' in 'field list' - errors.add("Unknown column "); - // java.sql.SQLException: Insert duplicate key on row: 4, pk: 138609795916627968 - errors.add("Insert duplicate key on row: "); - // com.mysql.cj.jdbc.exceptions.MysqlDataTruncation: Data truncation: Incorrect datetime value: - errors.add("Data truncation: Incorrect datetime value: "); - // java.sql.SQLException: Field 'c0' doesn't have a default value - errors.add("doesn't have a default value"); - // java.sql.SQLException: Data truncated for column 'c0' at row 1 - errors.addRegex(Pattern.compile("Data truncated for column 'c.*' at row .*")); - } - - private void appendPartition() { - - } - - private void appendColumnsAndValues(List columns) { - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(Randomly.fromOptions(" VALUES ", " VALUE ")); - appendValues(); - } - - private void appendAS() { - if (!globalState.getDbmsSpecificOptions().test80Version) { - return; - } - sb.append(" AS "); - sb.append("r").append(table.getNrRows(globalState)); - } - - private void appendOnDuplicateUpdate() { - sb.append("on duplicate key update "); - StoneDBColumn randomColumn = table.getRandomColumn(); - sb.append(randomColumn.getName()); - sb.append("="); - insertValue(randomColumn); - } - - // append nrRows rows - private void appendValues() { - int nrRows; - if (Randomly.getBoolean()) { - nrRows = 1; - } else { - nrRows = 1 + Randomly.smallNumber(); - } - for (int row = 0; row < nrRows; row++) { - appendOneValue(row); - } - } - - // append all columns of one row - private void appendOneValue(int nrRow) { - if (nrRow != 0) { - sb.append(", "); - } - sb.append("("); - for (int c = 0; c < columns.size(); c++) { - if (c != 0) { - sb.append(", "); - } - insertValue(columns.get(c)); - } - sb.append(")"); - } - - // append one column of one row - @Override - protected void insertValue(StoneDBColumn column) { - if (!bugNotReported5 && Randomly.getBooleanWithRatherLowProbability()) { - sb.append("DEFAULT"); - } else { - sb.append(StoneDBToStringVisitor.asString(new StoneDBExpressionGenerator(globalState) - .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); - } - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java deleted file mode 100644 index 259136135..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBTableUpdateGenerator.java +++ /dev/null @@ -1,71 +0,0 @@ -package sqlancer.stonedb.gen; - -import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBTableUpdateGenerator { - private final StoneDBGlobalState globalState; - // which table to insert into - private final StoneDBTable table; - private final StringBuilder sb = new StringBuilder(); - ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBTableUpdateGenerator(StoneDBGlobalState globalState) { - this.globalState = globalState; - table = globalState.getSchema().getRandomTable(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState) { - return new StoneDBTableUpdateGenerator(globalState).getQuery(); - } - - public SQLQueryAdapter getQuery() { - sb.append("UPDATE"); - sb.append(Randomly.fromOptions(" ", " LOW_PRIORITY ", " LOW_PRIORITY IGNORE ", " IGNORE ")); - sb.append(table.getName()); - sb.append("SET "); - appendAssignmentList(); - if (Randomly.getBoolean()) { - appendWhereCondition(); - } - if (Randomly.getBoolean()) { - appendOrderBy(); - } - if (Randomly.getBoolean()) { - appendLimit(); - } - return new SQLQueryAdapter(sb.toString(), errors); - } - - public void appendAssignmentList() { - for (int i = 0; i < new Randomly().getInteger(1, table.getColumns().size()); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(table.getColumns().get(i)); - sb.append(" = "); - sb.append(Randomly.fromOptions( - StoneDBToStringVisitor.asString(StoneDBSchema.StoneDBDataType.getRandomValue( - table.getColumns().get(i).getType().getPrimitiveDataType(), globalState.getRandomly())), - "DEFAULT")); - } - } - - private void appendWhereCondition() { - } - - private void appendOrderBy() { - sb.append(" ORDER BY "); - sb.append(table.getRandomColumn().getName()); - } - - private void appendLimit() { - sb.append(" LIMIT "); - sb.append(new Randomly().getInteger(0, (int) table.getNrRows(globalState))); - } -} diff --git a/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java b/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java deleted file mode 100644 index d4ada0332..000000000 --- a/src/sqlancer/stonedb/gen/StoneDBViewCreateGenerator.java +++ /dev/null @@ -1,78 +0,0 @@ -package sqlancer.stonedb.gen; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBSchema.StoneDBTables; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; - -public class StoneDBViewCreateGenerator { - // the name of the view to create - private final String viewName; - private final StoneDBGlobalState globalState; - private StoneDBSelect select; - private final StringBuilder sb = new StringBuilder(); - - public StoneDBViewCreateGenerator(StoneDBGlobalState globalState, String viewName) { - this.globalState = globalState; - this.viewName = viewName; - setSelect(); - } - - public static SQLQueryAdapter generate(StoneDBGlobalState globalState, String viewName) { - return new StoneDBViewCreateGenerator(globalState, viewName).getQuery(); - } - - private SQLQueryAdapter getQuery() { - sb.append("CREATE "); - sb.append("VIEW "); - sb.append(viewName); - sb.append(" AS "); - sb.append(StoneDBToStringVisitor.asString(select)); - return new SQLQueryAdapter(sb.toString(), true); - } - - private void setSelect() { - StoneDBSchema schema = globalState.getSchema(); - StoneDBTables targetTables = schema.getRandomTableNonEmptyTables(); - StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(globalState) - .setColumns(targetTables.getColumns()); - - select = new StoneDBSelect(); - select.setFetchColumns(generateFetchColumns(targetTables)); - List tables = targetTables.getTables(); - List> tableReferenceNodeList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = StoneDBJoin.getJoins(tableReferenceNodeList, globalState); - select.setJoinList(new ArrayList<>(joins)); - select.setFromList(new ArrayList<>(tableReferenceNodeList)); - select.setWhereClause(gen.generateExpression()); - select.setOrderByClauses(gen.generateOrderBys()); - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - } - - private List> generateFetchColumns(StoneDBTables tables) { - List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))); - } else { - columns = Randomly.nonEmptySubset(tables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)) - .collect(Collectors.toList()); - } - return columns; - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java b/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java deleted file mode 100644 index eb1482d7f..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBAggregateOracle.java +++ /dev/null @@ -1,170 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.stonedb.StoneDBErrors; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBCompositeDataType; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBAggregate.StoneDBAggregateFunction; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPostfixOperator; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBUnaryPrefixOperator; - -public class StoneDBAggregateOracle extends StoneDBQueryPartitioningBase { - - public StoneDBAggregateOracle(StoneDBGlobalState state) { - super(state); - StoneDBErrors.addExpectedExpressionErrors(state, errors); - } - - @Override - public void check() throws Exception { - super.check(); - - StoneDBAggregateFunction aggregateFunction = Randomly.fromOptions(StoneDBAggregateFunction.values()); - NewFunctionNode aggregate = gen - .generateAggregateAndArgs(aggregateFunction); - - List> fetchColumns = new ArrayList<>(); - fetchColumns.add(aggregate); - select.setFetchColumns(fetchColumns); - - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - - String originalQuery = StoneDBToStringVisitor.asString(select); - String originalResult = getAggregateResult(originalQuery); - - String metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); - String metamorphicResult = getAggregateResult(metamorphicQuery); - - String line1 = "--" + originalQuery + ";"; - String line2 = "--" + originalResult + ";"; - String line3 = "--" + originalResult + ";"; - String line4 = "--" + metamorphicResult + ";"; - String output = String.join(System.lineSeparator(), line1, line2, line3, line4); - state.getState().getLocalState().log(output); - - if (originalResult == null && metamorphicResult != null - || originalResult != null && (!originalResult.contentEquals(metamorphicResult) - && !ComparatorHelper.isEqualDouble(originalResult, metamorphicResult))) { - throw new AssertionError("aggregate result mismatch!" + System.lineSeparator() + output); - } - } - - private String createMetamorphicUnionQuery(StoneDBSelect select, - NewFunctionNode aggregate, - List> from) { - String metamorphicQuery; - Node whereClause = gen.generateExpression(); - - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, - StoneDBUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, - StoneDBUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); - StoneDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); - StoneDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); - StoneDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; - metamorphicQuery += StoneDBToStringVisitor.asString(leftSelect) + " UNION ALL " - + StoneDBToStringVisitor.asString(middleSelect) + " UNION ALL " - + StoneDBToStringVisitor.asString(rightSelect); - metamorphicQuery += ") as result"; - return metamorphicQuery; - } - - private String getAggregateResult(String queryString) throws SQLException { - String resultString; - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - try (SQLancerResultSet result = q.executeAndGet(state)) { - if (result == null) { - throw new IgnoreMeException(); - } - if (!result.next()) { - resultString = null; - } else { - resultString = result.getString(1); - } - return resultString; - } catch (SQLException e) { - if (!e.getMessage().contains("Not implemented type")) { - throw new AssertionError(queryString, e); - } else { - throw new IgnoreMeException(); - } - } - } - - private List> mapped( - NewFunctionNode aggregate) { - - StoneDBCastOperation count; - switch (aggregate.getFunc()) { - case COUNT: - case MAX: - case MIN: - case SUM: - return aliasArgs(List.of(aggregate)); - case AVG: - NewFunctionNode sum = new NewFunctionNode<>( - aggregate.getArgs(), StoneDBAggregateFunction.SUM); - count = new StoneDBCastOperation(new NewFunctionNode<>(aggregate.getArgs(), StoneDBAggregateFunction.COUNT), - new StoneDBCompositeDataType(StoneDBDataType.DECIMAL).getPrimitiveDataType()); - return aliasArgs(Arrays.asList(sum, count)); - default: - throw new AssertionError(aggregate.getFunc()); - } - } - - private List> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); - int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode(expr, "agg" + i++)); - } - return args; - } - - private String getOuterAggregateFunction(NewFunctionNode aggregate) { - switch (aggregate.getFunc()) { - case AVG: - return "SUM(agg0)/SUM(agg1)"; - case COUNT: - return StoneDBAggregateFunction.SUM + "(agg0)"; - default: - return aggregate.getFunc().toString() + "(agg0)"; - } - } - - private StoneDBSelect getSelect(List> aggregates, List> fromList, - Node whereClause, List> joinList) { - StoneDBSelect select = new StoneDBSelect(); - select.setFetchColumns(aggregates); - select.setFromList(fromList); - select.setWhereClause(whereClause); - select.setJoinList(joinList); - if (Randomly.getBooleanWithSmallProbability()) { - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - } - return select; - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java b/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java deleted file mode 100644 index ebf1f9967..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBFuzzOracle.java +++ /dev/null @@ -1,67 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.stonedb.StoneDBErrors; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBSchema.StoneDBTables; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator; - -public class StoneDBFuzzOracle implements TestOracle { - private final StoneDBGlobalState globalState; - private final StoneDBSchema schema; - private final ExpectedErrors errors = new ExpectedErrors(); - - public StoneDBFuzzOracle(StoneDBGlobalState globalState) { - this.globalState = globalState; - this.schema = globalState.getSchema(); - StoneDBErrors.addExpectedExpressionErrors(globalState, errors); - } - - @Override - public void check() throws Exception { - StoneDBTables randomTables = schema.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(globalState).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); - List> joins = StoneDBJoin.getJoins(tableList, globalState); - - StoneDBSelect select = new StoneDBSelect(); - select.setFetchColumns( - Randomly.fromOptions(List.of(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))), - allColumns, Randomly.subset(allColumns))); - select.setFromList(new ArrayList<>(tableList)); - select.setJoinList(joins); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses( - new StoneDBExpressionGenerator(globalState).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - - String queryString = StoneDBToStringVisitor.asString(select); - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); - q.executeAndGetLogged(globalState); - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java b/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java deleted file mode 100644 index d6efe30d1..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBNoRECOracle.java +++ /dev/null @@ -1,134 +0,0 @@ -package sqlancer.stonedb.oracle; - -import static sqlancer.stonedb.StoneDBBugs.bug1953; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.common.oracle.NoRECBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.stonedb.StoneDBErrors; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBDataType; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBSchema.StoneDBTables; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBBinaryLogicalOperator; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator.StoneDBCastOperation; - -public class StoneDBNoRECOracle extends NoRECBase implements TestOracle { - - private final StoneDBSchema schema; - - public StoneDBNoRECOracle(StoneDBGlobalState globalState) { - super(globalState); - this.schema = globalState.getSchema(); - StoneDBErrors.addExpectedExpressionErrors(globalState, errors); - } - - @Override - public void check() throws Exception { - StoneDBTables randomTables = schema.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - StoneDBExpressionGenerator gen = new StoneDBExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = StoneDBJoin.getJoins(tableList, state); - // get and check count - int secondCount = getUnoptimizedQueryCount(new ArrayList<>(tableList), randomWhereCondition, joins); - int firstCount = getOptimizedQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError(optimizedQueryString + "; -- " + firstCount + System.lineSeparator() - + unoptimizedQueryString + " -- " + secondCount); - } - } - - private int getUnoptimizedQueryCount(List> tableList, - Node randomWhereCondition, List> joins) throws SQLException { - StoneDBSelect select = new StoneDBSelect(); - Node asText = new NewPostfixTextNode<>( - new StoneDBCastOperation( - new NewBinaryOperatorNode<>(new NewPostfixTextNode<>(randomWhereCondition, " IS NOT NULL "), - randomWhereCondition, StoneDBBinaryLogicalOperator.AND), - StoneDBDataType.INT), - " as count"); - select.setFetchColumns(List.of(asText)); - select.setFromList(tableList); - select.setJoinList(joins); - unoptimizedQueryString = "SELECT SUM(count) FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; - if (bug1953) { - unoptimizedQueryString = "SELECT * FROM (" + StoneDBToStringVisitor.asString(select) + ") as res;"; - } - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError("error occurred when executing: \"" + unoptimizedQueryString + "\"", e); - } - if (rs == null) { - return -1; - } - int secondCount = 0; - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(SQLConnection con, List> tableList, - List columns, Node randomWhereCondition, - List> joins) { - StoneDBSelect select = new StoneDBSelect(); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new StoneDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = StoneDBToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java deleted file mode 100644 index 84d9f794f..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningBase.java +++ /dev/null @@ -1,71 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBSchema.StoneDBTable; -import sqlancer.stonedb.StoneDBSchema.StoneDBTables; -import sqlancer.stonedb.ast.StoneDBExpression; -import sqlancer.stonedb.ast.StoneDBJoin; -import sqlancer.stonedb.ast.StoneDBSelect; -import sqlancer.stonedb.gen.StoneDBExpressionGenerator; - -public class StoneDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, StoneDBGlobalState> - implements TestOracle { - - StoneDBSchema schema; - StoneDBTables targetTables; - StoneDBExpressionGenerator gen; - StoneDBSelect select; - - public StoneDBQueryPartitioningBase(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getRandomTableNonEmptyTables(); - gen = new StoneDBExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - - select = new StoneDBSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List> tableReferenceNodeList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = StoneDBJoin.getJoins(tableReferenceNodeList, state); - select.setJoinList(new ArrayList<>(joins)); - select.setFromList(new ArrayList<>(tableReferenceNodeList)); - select.setWhereClause(null); - } - - List> generateFetchColumns() { - List> columns = new ArrayList<>(); - if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new StoneDBColumn("*", null, false, false, 0))); - } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)) - .collect(Collectors.toList()); - } - return columns; - } - - @Override - protected ExpressionGenerator> getGen() { - return gen; - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java deleted file mode 100644 index cba8290b4..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningDistinctTester.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBQueryPartitioningDistinctTester extends StoneDBQueryPartitioningBase { - public StoneDBQueryPartitioningDistinctTester(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - - select.setDistinct(true); - select.setWhereClause(null); - String originalQueryString = StoneDBToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - if (Randomly.getBoolean()) { - select.setDistinct(false); - } - - select.setWhereClause(predicate); - String firstQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = StoneDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, - secondQueryString, thirdQueryString, combinedString, true, state, errors); - - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java deleted file mode 100644 index d7674a71c..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningGroupByTester.java +++ /dev/null @@ -1,50 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBSchema.StoneDBColumn; -import sqlancer.stonedb.StoneDBToStringVisitor; -import sqlancer.stonedb.ast.StoneDBExpression; - -public class StoneDBQueryPartitioningGroupByTester extends StoneDBQueryPartitioningBase { - public StoneDBQueryPartitioningGroupByTester(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - // common to both original and combined query string - select.setGroupByExpressions(select.getFetchColumns()); - // specific to original query string - select.setWhereClause(null); - String originalQueryString = StoneDBToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - // specific to combined query string, will produce the same result as original query string in logic - select.setWhereClause(predicate); - String firstQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = StoneDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, - secondQueryString, thirdQueryString, combinedString, true, state, errors); - // compare the result - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } - - @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java deleted file mode 100644 index ce9dd6fc7..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningHavingTester.java +++ /dev/null @@ -1,47 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBQueryPartitioningHavingTester extends StoneDBQueryPartitioningBase { - public StoneDBQueryPartitioningHavingTester(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - // common to both original and combined query string - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); - } - boolean orderBy = Randomly.getBoolean(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - // specific to original query string - select.setHavingClause(null); - String originalQueryString = StoneDBToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - // specific to combined query string, will produce the same result as original - // query string in logic - select.setHavingClause(predicate); - String firstQueryString = StoneDBToStringVisitor.asString(select); - select.setHavingClause(negatedPredicate); - String secondQueryString = StoneDBToStringVisitor.asString(select); - select.setHavingClause(isNullPredicate); - String thirdQueryString = StoneDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - // compare the result - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } -} diff --git a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java b/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java deleted file mode 100644 index 4c9267f68..000000000 --- a/src/sqlancer/stonedb/oracle/StoneDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.stonedb.oracle; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.stonedb.StoneDBProvider.StoneDBGlobalState; -import sqlancer.stonedb.StoneDBToStringVisitor; - -public class StoneDBQueryPartitioningWhereTester extends StoneDBQueryPartitioningBase { - public StoneDBQueryPartitioningWhereTester(StoneDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - - select.setWhereClause(null); - String originalQueryString = StoneDBToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = StoneDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = StoneDBToStringVisitor.asString(select); - - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - // compare the result - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } -} diff --git a/test/sqlancer/dbms/TestStoneDBFuzz.java b/test/sqlancer/dbms/TestStoneDBFuzz.java deleted file mode 100644 index 2f5d52424..000000000 --- a/test/sqlancer/dbms/TestStoneDBFuzz.java +++ /dev/null @@ -1,21 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestStoneDBFuzz { - - String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); - boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); - - @Test - public void testStoneDB() { - assumeTrue(stoneDBIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", - "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "FUZZER")); - } - -} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestStoneDBNoRec.java b/test/sqlancer/dbms/TestStoneDBNoRec.java deleted file mode 100644 index 4b158f372..000000000 --- a/test/sqlancer/dbms/TestStoneDBNoRec.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestStoneDBNoRec { - - String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); - boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); - - @Test - public void testStoneDB() { - assumeTrue(stoneDBIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", - "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "NoREC")); - } - -} diff --git a/test/sqlancer/dbms/TestStoneDBTLP.java b/test/sqlancer/dbms/TestStoneDBTLP.java deleted file mode 100644 index d5db5b9ab..000000000 --- a/test/sqlancer/dbms/TestStoneDBTLP.java +++ /dev/null @@ -1,21 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestStoneDBTLP { - - String stoneDBAvailable = System.getenv("STONEDB_AVAILABLE"); - boolean stoneDBIsAvailable = stoneDBAvailable != null && stoneDBAvailable.equalsIgnoreCase("true"); - - @Test - public void testStoneDB() { - assumeTrue(stoneDBIsAvailable); - assertEquals(0, Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", - "1", "--num-queries", TestConfig.NUM_QUERIES, "stonedb", "--oracle", "QUERY_PARTITIONING")); - } - -} From 5652c9db60349eda399275496ab57a39b3bda6b0 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 21 Aug 2024 15:47:20 +0800 Subject: [PATCH 468/856] Update new AST to support common test oracles --- .../common/ast/newast/NewAliasNode.java | 6 +++--- .../ast/newast/NewBetweenOperatorNode.java | 14 +++++++------- .../ast/newast/NewBinaryOperatorNode.java | 10 +++++----- .../ast/newast/NewCaseOperatorNode.java | 19 +++++++++---------- .../common/ast/newast/NewFunctionNode.java | 6 +++--- .../common/ast/newast/NewInOperatorNode.java | 10 +++++----- .../common/ast/newast/NewOrderingTerm.java | 6 +++--- .../common/ast/newast/NewPostfixTextNode.java | 6 +++--- .../common/ast/newast/NewTernaryNode.java | 14 +++++++------- .../common/ast/newast/NewToStringVisitor.java | 6 +++--- .../newast/NewUnaryPostfixOperatorNode.java | 7 +++---- .../newast/NewUnaryPrefixOperatorNode.java | 6 +++--- 12 files changed, 54 insertions(+), 56 deletions(-) diff --git a/src/sqlancer/common/ast/newast/NewAliasNode.java b/src/sqlancer/common/ast/newast/NewAliasNode.java index 9d7d84c43..9fbbb35d5 100644 --- a/src/sqlancer/common/ast/newast/NewAliasNode.java +++ b/src/sqlancer/common/ast/newast/NewAliasNode.java @@ -2,15 +2,15 @@ public class NewAliasNode implements Node { - private final Node expr; + private final E expr; private final String alias; - public NewAliasNode(Node expr, String alias) { + public NewAliasNode(E expr, String alias) { this.expr = expr; this.alias = alias; } - public Node getExpr() { + public E getExpr() { return expr; } diff --git a/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java index c5a0acf27..e47012267 100644 --- a/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java @@ -2,27 +2,27 @@ public class NewBetweenOperatorNode implements Node { - protected Node left; - protected Node middle; - protected Node right; + protected T left; + protected T middle; + protected T right; protected boolean isTrue; - public NewBetweenOperatorNode(Node left, Node middle, Node right, boolean isTrue) { + public NewBetweenOperatorNode(T left, T middle, T right, boolean isTrue) { this.left = left; this.middle = middle; this.right = right; this.isTrue = isTrue; } - public Node getLeft() { + public T getLeft() { return left; } - public Node getMiddle() { + public T getMiddle() { return middle; } - public Node getRight() { + public T getRight() { return right; } diff --git a/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java index 2b640ed0c..6f150499c 100644 --- a/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java @@ -5,10 +5,10 @@ public class NewBinaryOperatorNode implements Node { protected final Operator op; - protected final Node left; - protected final Node right; + protected final T left; + protected final T right; - public NewBinaryOperatorNode(Node left, Node right, Operator op) { + public NewBinaryOperatorNode(T left, T right, Operator op) { this.left = left; this.right = right; this.op = op; @@ -18,11 +18,11 @@ public String getOperatorRepresentation() { return op.getTextRepresentation(); } - public Node getLeft() { + public T getLeft() { return left; } - public Node getRight() { + public T getRight() { return right; } diff --git a/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java index 04f6be4c5..5ab2ed853 100644 --- a/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java @@ -4,13 +4,12 @@ public class NewCaseOperatorNode implements Node { - private final List> conditions; - private final List> expressions; - private final Node elseExpr; - private final Node switchCondition; + private final List conditions; + private final List expressions; + private final T elseExpr; + private final T switchCondition; - public NewCaseOperatorNode(Node switchCondition, List> conditions, List> expressions, - Node elseExpr) { + public NewCaseOperatorNode(T switchCondition, List conditions, List expressions, T elseExpr) { this.switchCondition = switchCondition; this.conditions = conditions; this.expressions = expressions; @@ -20,19 +19,19 @@ public NewCaseOperatorNode(Node switchCondition, List> conditions, Li } } - public Node getSwitchCondition() { + public T getSwitchCondition() { return switchCondition; } - public List> getConditions() { + public List getConditions() { return conditions; } - public List> getExpressions() { + public List getExpressions() { return expressions; } - public Node getElseExpr() { + public T getElseExpr() { return elseExpr; } diff --git a/src/sqlancer/common/ast/newast/NewFunctionNode.java b/src/sqlancer/common/ast/newast/NewFunctionNode.java index fae214713..baa5d1bc3 100644 --- a/src/sqlancer/common/ast/newast/NewFunctionNode.java +++ b/src/sqlancer/common/ast/newast/NewFunctionNode.java @@ -4,15 +4,15 @@ public class NewFunctionNode implements Node { - protected List> args; + protected List args; protected F func; - public NewFunctionNode(List> args, F func) { + public NewFunctionNode(List args, F func) { this.args = args; this.func = func; } - public List> getArgs() { + public List getArgs() { return args; } diff --git a/src/sqlancer/common/ast/newast/NewInOperatorNode.java b/src/sqlancer/common/ast/newast/NewInOperatorNode.java index 346156666..b4bcf535d 100644 --- a/src/sqlancer/common/ast/newast/NewInOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewInOperatorNode.java @@ -4,21 +4,21 @@ public class NewInOperatorNode implements Node { - private final Node left; - private final List> right; + private final T left; + private final List right; private final boolean isNegated; - public NewInOperatorNode(Node left, List> right, boolean isNegated) { + public NewInOperatorNode(T left, List right, boolean isNegated) { this.left = left; this.right = right; this.isNegated = isNegated; } - public Node getLeft() { + public T getLeft() { return left; } - public List> getRight() { + public List getRight() { return right; } diff --git a/src/sqlancer/common/ast/newast/NewOrderingTerm.java b/src/sqlancer/common/ast/newast/NewOrderingTerm.java index 5afe71821..be938fbb4 100644 --- a/src/sqlancer/common/ast/newast/NewOrderingTerm.java +++ b/src/sqlancer/common/ast/newast/NewOrderingTerm.java @@ -4,7 +4,7 @@ public class NewOrderingTerm implements Node { - private final Node expr; + private final T expr; private final Ordering ordering; public enum Ordering { @@ -15,12 +15,12 @@ public static Ordering getRandom() { } } - public NewOrderingTerm(Node expr, Ordering ordering) { + public NewOrderingTerm(T expr, Ordering ordering) { this.expr = expr; this.ordering = ordering; } - public Node getExpr() { + public T getExpr() { return expr; } diff --git a/src/sqlancer/common/ast/newast/NewPostfixTextNode.java b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java index 70e65b17b..4d03a300c 100644 --- a/src/sqlancer/common/ast/newast/NewPostfixTextNode.java +++ b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java @@ -2,15 +2,15 @@ public class NewPostfixTextNode implements Node { - private final Node expr; + private final T expr; private final String text; - public NewPostfixTextNode(Node expr, String text) { + public NewPostfixTextNode(T expr, String text) { this.expr = expr; this.text = text; } - public Node getExpr() { + public T getExpr() { return expr; } diff --git a/src/sqlancer/common/ast/newast/NewTernaryNode.java b/src/sqlancer/common/ast/newast/NewTernaryNode.java index f40c480ec..ab884e48c 100644 --- a/src/sqlancer/common/ast/newast/NewTernaryNode.java +++ b/src/sqlancer/common/ast/newast/NewTernaryNode.java @@ -2,13 +2,13 @@ public class NewTernaryNode implements Node { - protected final Node left; - protected final Node middle; - protected final Node right; + protected final T left; + protected final T middle; + protected final T right; private final String leftStr; private final String rightStr; - public NewTernaryNode(Node left, Node middle, Node right, String leftStr, String rightStr) { + public NewTernaryNode(T left, T middle, T right, String leftStr, String rightStr) { this.left = left; this.middle = middle; this.right = right; @@ -16,15 +16,15 @@ public NewTernaryNode(Node left, Node middle, Node right, String leftSt this.rightStr = rightStr; } - public Node getLeft() { + public T getLeft() { return left; } - public Node getMiddle() { + public T getMiddle() { return middle; } - public Node getRight() { + public T getRight() { return right; } diff --git a/src/sqlancer/common/ast/newast/NewToStringVisitor.java b/src/sqlancer/common/ast/newast/NewToStringVisitor.java index 3ad625440..82b6bace2 100644 --- a/src/sqlancer/common/ast/newast/NewToStringVisitor.java +++ b/src/sqlancer/common/ast/newast/NewToStringVisitor.java @@ -7,7 +7,7 @@ public abstract class NewToStringVisitor { protected final StringBuilder sb = new StringBuilder(); @SuppressWarnings("unchecked") - public void visit(Node expr) { + public void visit(E expr) { assert expr != null; if (expr instanceof ColumnReferenceNode) { sb.append(((ColumnReferenceNode) expr).getColumn().getFullQualifiedName()); @@ -40,7 +40,7 @@ public void visit(Node expr) { } } - public void visit(List> expressions) { + public void visit(List expressions) { for (int i = 0; i < expressions.size(); i++) { if (i != 0) { sb.append(", "); @@ -165,6 +165,6 @@ public String get() { return sb.toString(); } - public abstract void visitSpecific(Node expr); + public abstract void visitSpecific(E expr); } diff --git a/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java index ebd17945f..fb7c28a0f 100644 --- a/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java @@ -5,9 +5,9 @@ public class NewUnaryPostfixOperatorNode implements Node { protected final Operator op; - private final Node expr; + private final T expr; - public NewUnaryPostfixOperatorNode(Node expr, Operator op) { + public NewUnaryPostfixOperatorNode(T expr, Operator op) { this.expr = expr; this.op = op; } @@ -16,8 +16,7 @@ public String getOperatorRepresentation() { return op.getTextRepresentation(); } - public Node getExpr() { + public T getExpr() { return expr; } - } diff --git a/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java index 5274bd942..df75140f7 100644 --- a/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java @@ -5,9 +5,9 @@ public class NewUnaryPrefixOperatorNode implements Node { protected final Operator op; - private final Node expr; + private final T expr; - public NewUnaryPrefixOperatorNode(Node expr, Operator op) { + public NewUnaryPrefixOperatorNode(T expr, Operator op) { this.expr = expr; this.op = op; } @@ -16,7 +16,7 @@ public String getOperatorRepresentation() { return op.getTextRepresentation(); } - public Node getExpr() { + public T getExpr() { return expr; } From f401c57199013ba671be69ded2692322abe1549b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 21 Aug 2024 15:48:51 +0800 Subject: [PATCH 469/856] [DuckDB] Update expressions to match new AST --- .../duckdb/DuckDBToStringVisitor.java | 10 +- src/sqlancer/duckdb/ast/DuckDBAlias.java | 9 ++ .../duckdb/ast/DuckDBBetweenOperator.java | 10 ++ .../duckdb/ast/DuckDBBinaryOperator.java | 10 ++ .../duckdb/ast/DuckDBCaseOperator.java | 12 +++ .../duckdb/ast/DuckDBColumnReference.java | 12 +++ src/sqlancer/duckdb/ast/DuckDBConstant.java | 18 ++-- src/sqlancer/duckdb/ast/DuckDBExpression.java | 4 +- src/sqlancer/duckdb/ast/DuckDBFunction.java | 11 +++ src/sqlancer/duckdb/ast/DuckDBInOperator.java | 11 +++ src/sqlancer/duckdb/ast/DuckDBJoin.java | 51 +++++----- .../duckdb/ast/DuckDBOrderingTerm.java | 9 ++ .../duckdb/ast/DuckDBPostFixText.java | 9 ++ src/sqlancer/duckdb/ast/DuckDBSelect.java | 3 +- .../duckdb/ast/DuckDBTableReference.java | 11 +++ src/sqlancer/duckdb/ast/DuckDBTernary.java | 10 ++ .../ast/DuckDBUnaryPostfixOperator.java | 11 +++ .../duckdb/ast/DuckDBUnaryPrefixOperator.java | 11 +++ .../duckdb/gen/DuckDBExpressionGenerator.java | 92 +++++++++---------- .../duckdb/gen/DuckDBIndexGenerator.java | 3 +- .../gen/DuckDBRandomQuerySynthesizer.java | 13 ++- .../duckdb/gen/DuckDBTableGenerator.java | 5 +- .../duckdb/gen/DuckDBUpdateGenerator.java | 3 +- .../duckdb/test/DuckDBNoRECOracle.java | 32 +++---- ...uckDBQueryPartitioningAggregateTester.java | 53 +++++------ .../test/DuckDBQueryPartitioningBase.java | 26 +++--- .../DuckDBQueryPartitioningGroupByTester.java | 10 +- .../DuckDBQueryPartitioningHavingTester.java | 5 +- 28 files changed, 289 insertions(+), 175 deletions(-) create mode 100644 src/sqlancer/duckdb/ast/DuckDBAlias.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBCaseOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBColumnReference.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBFunction.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBInOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBPostFixText.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBTableReference.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBTernary.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index 37493d0f6..b1a8eb703 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -1,7 +1,7 @@ package sqlancer.duckdb; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; +import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.duckdb.ast.DuckDBConstant; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; @@ -10,7 +10,7 @@ public class DuckDBToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(DuckDBExpression expr) { if (expr instanceof DuckDBConstant) { visit((DuckDBConstant) expr); } else if (expr instanceof DuckDBSelect) { @@ -23,7 +23,7 @@ public void visitSpecific(Node expr) { } private void visit(DuckDBJoin join) { - visit(join.getLeftTable()); + visit((TableReferenceNode) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" "); @@ -31,7 +31,7 @@ private void visit(DuckDBJoin join) { sb.append(join.getOuterType()); } sb.append(" JOIN "); - visit(join.getRightTable()); + visit((TableReferenceNode) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); @@ -82,7 +82,7 @@ private void visit(DuckDBSelect select) { } } - public static String asString(Node expr) { + public static String asString(DuckDBExpression expr) { DuckDBToStringVisitor visitor = new DuckDBToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/duckdb/ast/DuckDBAlias.java b/src/sqlancer/duckdb/ast/DuckDBAlias.java new file mode 100644 index 000000000..cc1a91921 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBAlias.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class DuckDBAlias extends NewAliasNode implements DuckDBExpression { + public DuckDBAlias(DuckDBExpression expr, String string) { + super(expr, string); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java b/src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java new file mode 100644 index 000000000..eabaa6162 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBBetweenOperator.java @@ -0,0 +1,10 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class DuckDBBetweenOperator extends NewBetweenOperatorNode implements DuckDBExpression { + public DuckDBBetweenOperator(DuckDBExpression left, DuckDBExpression middle, DuckDBExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java b/src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java new file mode 100644 index 000000000..24492c145 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBBinaryOperator.java @@ -0,0 +1,10 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DuckDBBinaryOperator extends NewBinaryOperatorNode implements DuckDBExpression { + public DuckDBBinaryOperator(DuckDBExpression left, DuckDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBCaseOperator.java b/src/sqlancer/duckdb/ast/DuckDBCaseOperator.java new file mode 100644 index 000000000..ba1d9d96f --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBCaseOperator.java @@ -0,0 +1,12 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class DuckDBCaseOperator extends NewCaseOperatorNode implements DuckDBExpression { + public DuckDBCaseOperator(DuckDBExpression switchCondition, List conditions, + List expressions, DuckDBExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBColumnReference.java b/src/sqlancer/duckdb/ast/DuckDBColumnReference.java new file mode 100644 index 000000000..d24382777 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.duckdb.DuckDBSchema; + +public class DuckDBColumnReference extends ColumnReferenceNode + implements DuckDBExpression { + public DuckDBColumnReference(DuckDBSchema.DuckDBColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/duckdb/ast/DuckDBConstant.java b/src/sqlancer/duckdb/ast/DuckDBConstant.java index b895a18e5..f4af6918a 100644 --- a/src/sqlancer/duckdb/ast/DuckDBConstant.java +++ b/src/sqlancer/duckdb/ast/DuckDBConstant.java @@ -3,9 +3,7 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.common.ast.newast.Node; - -public class DuckDBConstant implements Node { +public class DuckDBConstant implements DuckDBExpression { private DuckDBConstant() { } @@ -161,31 +159,31 @@ public String toString() { } - public static Node createStringConstant(String text) { + public static DuckDBExpression createStringConstant(String text) { return new DuckDBTextConstant(text); } - public static Node createFloatConstant(double val) { + public static DuckDBExpression createFloatConstant(double val) { return new DuckDBDoubleConstant(val); } - public static Node createIntConstant(long val) { + public static DuckDBExpression createIntConstant(long val) { return new DuckDBIntConstant(val); } - public static Node createNullConstant() { + public static DuckDBExpression createNullConstant() { return new DuckDBNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static DuckDBExpression createBooleanConstant(boolean val) { return new DuckDBBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static DuckDBExpression createDateConstant(long integer) { return new DuckDBDateConstant(integer); } - public static Node createTimestampConstant(long integer) { + public static DuckDBExpression createTimestampConstant(long integer) { return new DuckDBTimestampConstant(integer); } diff --git a/src/sqlancer/duckdb/ast/DuckDBExpression.java b/src/sqlancer/duckdb/ast/DuckDBExpression.java index 3ff66ce33..21d4e6eb5 100644 --- a/src/sqlancer/duckdb/ast/DuckDBExpression.java +++ b/src/sqlancer/duckdb/ast/DuckDBExpression.java @@ -1,5 +1,7 @@ package sqlancer.duckdb.ast; -public interface DuckDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; +public interface DuckDBExpression extends Expression { } diff --git a/src/sqlancer/duckdb/ast/DuckDBFunction.java b/src/sqlancer/duckdb/ast/DuckDBFunction.java new file mode 100644 index 000000000..445583090 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBFunction.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DuckDBFunction extends NewFunctionNode implements DuckDBExpression { + public DuckDBFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBInOperator.java b/src/sqlancer/duckdb/ast/DuckDBInOperator.java new file mode 100644 index 000000000..619601749 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBInOperator.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class DuckDBInOperator extends NewInOperatorNode implements DuckDBExpression { + public DuckDBInOperator(DuckDBExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBJoin.java b/src/sqlancer/duckdb/ast/DuckDBJoin.java index c1c0c94a9..82031d1e7 100644 --- a/src/sqlancer/duckdb/ast/DuckDBJoin.java +++ b/src/sqlancer/duckdb/ast/DuckDBJoin.java @@ -4,19 +4,18 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.common.ast.newast.Join; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -public class DuckDBJoin implements Node { +public class DuckDBJoin implements DuckDBExpression, Join { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final DuckDBTableReference leftTable; + private final DuckDBTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private DuckDBExpression onCondition; private OuterType outerType; public enum JoinType { @@ -35,20 +34,19 @@ public static OuterType getRandom() { } } - public DuckDBJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public DuckDBJoin(DuckDBTableReference leftTable, DuckDBTableReference rightTable, JoinType joinType, + DuckDBExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public TableReferenceNode getLeftTable() { + public DuckDBTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public DuckDBTableReference getRightTable() { return rightTable; } @@ -56,7 +54,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public DuckDBExpression getOnCondition() { return onCondition; } @@ -68,12 +66,11 @@ public OuterType getOuterType() { return outerType; } - public static List> getJoins( - List> tableList, DuckDBGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DuckDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + DuckDBTableReference leftTable = tableList.remove(0); + DuckDBTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); DuckDBExpressionGenerator joinGen = new DuckDBExpressionGenerator(globalState).setColumns(columns); @@ -99,26 +96,30 @@ public static List> getJoins( return joinExpressions; } - public static DuckDBJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DuckDBJoin createRightOuterJoin(DuckDBTableReference left, DuckDBTableReference right, + DuckDBExpression predicate) { return new DuckDBJoin(left, right, JoinType.RIGHT, predicate); } - public static DuckDBJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DuckDBJoin createLeftOuterJoin(DuckDBTableReference left, DuckDBTableReference right, + DuckDBExpression predicate) { return new DuckDBJoin(left, right, JoinType.LEFT, predicate); } - public static DuckDBJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DuckDBJoin createInnerJoin(DuckDBTableReference left, DuckDBTableReference right, + DuckDBExpression predicate) { return new DuckDBJoin(left, right, JoinType.INNER, predicate); } - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, OuterType naturalJoinType) { + public static DuckDBExpression createNaturalJoin(DuckDBTableReference left, DuckDBTableReference right, + OuterType naturalJoinType) { DuckDBJoin join = new DuckDBJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; } + @Override + public void setOnClause(DuckDBExpression onClause) { + this.onCondition = onClause; + } } diff --git a/src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java b/src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java new file mode 100644 index 000000000..8d7177655 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBOrderingTerm.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class DuckDBOrderingTerm extends NewOrderingTerm implements DuckDBExpression { + public DuckDBOrderingTerm(DuckDBExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBPostFixText.java b/src/sqlancer/duckdb/ast/DuckDBPostFixText.java new file mode 100644 index 000000000..a877a4c92 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBPostFixText.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class DuckDBPostFixText extends NewPostfixTextNode implements DuckDBExpression { + public DuckDBPostFixText(DuckDBExpression expr, String string) { + super(expr, string); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBSelect.java b/src/sqlancer/duckdb/ast/DuckDBSelect.java index 9a2391aa3..10d76fbbd 100644 --- a/src/sqlancer/duckdb/ast/DuckDBSelect.java +++ b/src/sqlancer/duckdb/ast/DuckDBSelect.java @@ -1,9 +1,8 @@ package sqlancer.duckdb.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class DuckDBSelect extends SelectBase> implements Node { +public class DuckDBSelect extends SelectBase implements DuckDBExpression { private boolean isDistinct; diff --git a/src/sqlancer/duckdb/ast/DuckDBTableReference.java b/src/sqlancer/duckdb/ast/DuckDBTableReference.java new file mode 100644 index 000000000..0a8d795c8 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.duckdb.DuckDBSchema; + +public class DuckDBTableReference extends TableReferenceNode + implements DuckDBExpression { + public DuckDBTableReference(DuckDBSchema.DuckDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBTernary.java b/src/sqlancer/duckdb/ast/DuckDBTernary.java new file mode 100644 index 000000000..921f77b90 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBTernary.java @@ -0,0 +1,10 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewTernaryNode; + +public class DuckDBTernary extends NewTernaryNode implements DuckDBExpression { + public DuckDBTernary(DuckDBExpression left, DuckDBExpression middle, DuckDBExpression right, String leftString, + String rightString) { + super(left, middle, right, leftString, rightString); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java b/src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java new file mode 100644 index 000000000..856c129ec --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBUnaryPostfixOperator.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class DuckDBUnaryPostfixOperator extends NewUnaryPostfixOperatorNode + implements DuckDBExpression { + public DuckDBUnaryPostfixOperator(DuckDBExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java b/src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java new file mode 100644 index 000000000..95c8b4990 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBUnaryPrefixOperator.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class DuckDBUnaryPrefixOperator extends NewUnaryPrefixOperatorNode + implements DuckDBExpression { + public DuckDBUnaryPrefixOperator(DuckDBExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index eeba94027..49a390c07 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -7,27 +7,25 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewCaseOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; -import sqlancer.common.ast.newast.NewTernaryNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; +import sqlancer.duckdb.ast.DuckDBBetweenOperator; +import sqlancer.duckdb.ast.DuckDBBinaryOperator; +import sqlancer.duckdb.ast.DuckDBCaseOperator; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBConstant; import sqlancer.duckdb.ast.DuckDBExpression; +import sqlancer.duckdb.ast.DuckDBFunction; +import sqlancer.duckdb.ast.DuckDBInOperator; +import sqlancer.duckdb.ast.DuckDBOrderingTerm; +import sqlancer.duckdb.ast.DuckDBTernary; -public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator, DuckDBColumn> { +public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator { private final DuckDBGlobalState globalState; @@ -41,14 +39,14 @@ private enum Expression { } @Override - protected Node generateExpression(int depth) { + protected DuckDBExpression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); } if (allowAggregates && Randomly.getBoolean()) { DuckDBAggregateFunction aggregate = DuckDBAggregateFunction.getRandom(); allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + return new DuckDBFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); if (!globalState.getDbmsSpecificOptions().testCollate) { @@ -78,44 +76,41 @@ protected Node generateExpression(int depth) { Expression expr = Randomly.fromList(possibleOptions); switch (expr) { case COLLATE: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(generateExpression(depth + 1), DuckDBCollate.getRandom()); case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), + return new sqlancer.duckdb.ast.DuckDBUnaryPrefixOperator(generateExpression(depth + 1), DuckDBUnaryPrefixOperator.getRandom()); case UNARY_POSTFIX: - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), + return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(generateExpression(depth + 1), DuckDBUnaryPostfixOperator.getRandom()); case BINARY_COMPARISON: Operator op = DuckDBBinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new DuckDBBinaryOperator(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_LOGICAL: op = DuckDBBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new DuckDBBinaryOperator(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), DuckDBBinaryArithmeticOperator.getRandom()); + return new DuckDBBinaryOperator(generateExpression(depth + 1), generateExpression(depth + 1), + DuckDBBinaryArithmeticOperator.getRandom()); case CAST: return new DuckDBCastOperation(generateExpression(depth + 1), DuckDBCompositeDataType.getRandomWithoutNull()); case FUNC: DBFunction func = DBFunction.getRandom(); - return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + return new DuckDBFunction<>(generateExpressions(func.getNrArgs()), func); case BETWEEN: - return new NewBetweenOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + return new DuckDBBetweenOperator(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); case IN: - return new NewInOperatorNode(generateExpression(depth + 1), + return new DuckDBInOperator(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; - return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), - generateExpression(depth + 1)); + return new DuckDBCaseOperator(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); case LIKE_ESCAPE: - return new NewTernaryNode(generateExpression(depth + 1), generateExpression(depth + 1), + return new DuckDBTernary(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1), "LIKE", "ESCAPE"); default: throw new AssertionError(); @@ -123,13 +118,13 @@ protected Node generateExpression(int depth) { } @Override - protected Node generateColumn() { + protected DuckDBExpression generateColumn() { DuckDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode(column); + return new DuckDBColumnReference(column); } @Override - public Node generateConstant() { + public DuckDBExpression generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return DuckDBConstant.createNullConstant(); } @@ -171,21 +166,22 @@ public Node generateConstant() { } @Override - public List> generateOrderBys() { - List> expr = super.generateOrderBys(); - List> newExpr = new ArrayList<>(expr.size()); - for (Node curExpr : expr) { + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (DuckDBExpression curExpr : expr) { if (Randomly.getBoolean()) { - curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); + curExpr = new DuckDBOrderingTerm(curExpr, Ordering.getRandom()); } newExpr.add(curExpr); } return newExpr; }; - public static class DuckDBCastOperation extends NewUnaryPostfixOperatorNode { + public static class DuckDBCastOperation extends NewUnaryPostfixOperatorNode + implements DuckDBExpression { - public DuckDBCastOperation(Node expr, DuckDBCompositeDataType type) { + public DuckDBCastOperation(DuckDBExpression expr, DuckDBCompositeDataType type) { super(expr, new Operator() { @Override @@ -422,25 +418,23 @@ public String getTextRepresentation() { } - public NewFunctionNode generateArgsForAggregate( - DuckDBAggregateFunction aggregateFunction) { - return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + public DuckDBFunction generateArgsForAggregate(DuckDBAggregateFunction aggregateFunction) { + return new DuckDBFunction<>(generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); } - public Node generateAggregate() { + public DuckDBExpression generateAggregate() { DuckDBAggregateFunction aggrFunc = DuckDBAggregateFunction.getRandom(); return generateArgsForAggregate(aggrFunc); } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, DuckDBUnaryPrefixOperator.NOT); + public DuckDBExpression negatePredicate(DuckDBExpression predicate) { + return new sqlancer.duckdb.ast.DuckDBUnaryPrefixOperator(predicate, DuckDBUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, DuckDBUnaryPostfixOperator.IS_NULL); + public DuckDBExpression isNull(DuckDBExpression expr) { + return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(expr, DuckDBUnaryPostfixOperator.IS_NULL); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index eec5384aa..daa22da5f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -45,7 +44,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(")"); if (Randomly.getBoolean()) { sb.append(" WHERE "); - Node expr = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()) + DuckDBExpression expr = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(); sb.append(DuckDBToStringVisitor.asString(expr)); } diff --git a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java index aef88ef13..c4c0d7c95 100644 --- a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java +++ b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java @@ -5,8 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBSchema.DuckDBTables; @@ -14,6 +12,7 @@ import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; public final class DuckDBRandomQuerySynthesizer { @@ -28,10 +27,10 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC // TODO: distinct // select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = gen.generateExpression(); + DuckDBExpression expression = gen.generateExpression(); columns.add(expression); // } else { // columns.add(gen()); @@ -39,9 +38,9 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC } select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + List joins = DuckDBJoin.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index d3cd27fde..ea6d3537f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -5,7 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -27,8 +26,8 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(tableName); sb.append("("); List columns = getNewColumns(); - UntypedExpressionGenerator, DuckDBColumn> gen = new DuckDBExpressionGenerator( - globalState).setColumns(columns); + UntypedExpressionGenerator gen = new DuckDBExpressionGenerator(globalState) + .setColumns(columns); for (int i = 0; i < columns.size(); i++) { if (i != 0) { sb.append(", "); diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 13b7f9d74..3bb22b2ea 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; @@ -40,7 +39,7 @@ private SQLQueryAdapter generate() { @Override protected void updateValue(DuckDBColumn column) { - Node expr; + DuckDBExpression expr; if (Randomly.getBooleanWithSmallProbability()) { expr = gen.generateExpression(); DuckDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 2ce6a8ad2..63932492d 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -10,10 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -27,9 +23,12 @@ import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBSchema.DuckDBTables; import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; +import sqlancer.duckdb.ast.DuckDBPostFixText; import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBCastOperation; @@ -48,11 +47,11 @@ public void check() throws SQLException { DuckDBTables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generateExpression(); + DuckDBExpression randomWhereCondition = gen.generateExpression(); List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + List joins = DuckDBJoin.getJoins(tableList, state); int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, randomWhereCondition, joins); @@ -65,14 +64,14 @@ public void check() throws SQLException { } } - private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + private int getSecondQuery(List tableList, DuckDBExpression randomWhereCondition, + List joins) throws SQLException { DuckDBSelect select = new DuckDBSelect(); // select.setGroupByClause(groupBys); // DuckDBExpression isTrue = DuckDBPostfixOperation.create(randomWhereCondition, // PostfixOperator.IS_TRUE); - Node asText = new NewPostfixTextNode<>(new DuckDBCastOperation( - new NewPostfixTextNode(randomWhereCondition, + DuckDBExpression asText = new DuckDBPostFixText(new DuckDBCastOperation( + new DuckDBPostFixText(randomWhereCondition, " IS NOT NULL AND " + DuckDBToStringVisitor.asString(randomWhereCondition)), new DuckDBCompositeDataType(DuckDBDataType.INT, 8)), "as count"); select.setFetchColumns(Arrays.asList(asText)); @@ -99,14 +98,13 @@ private int getSecondQuery(List> tableList, Node> tableList, - List columns, Node randomWhereCondition, List> joins) - throws SQLException { + private int getFirstQueryCount(SQLConnection con, List tableList, List columns, + DuckDBExpression randomWhereCondition, List joins) throws SQLException { DuckDBSelect select = new DuckDBSelect(); // select.setGroupByClause(groupBys); // DuckDBAggregate aggr = new DuckDBAggregate( - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List allColumns = columns.stream().map((c) -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); // DuckDBAggregateFunction.COUNT); // select.setFetchColumns(Arrays.asList(aggr)); select.setFetchColumns(allColumns); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java index de8bd96a0..e82f97db6 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningAggregateTester.java @@ -8,12 +8,7 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; @@ -22,7 +17,10 @@ import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.ast.DuckDBAlias; +import sqlancer.duckdb.ast.DuckDBBinaryOperator; import sqlancer.duckdb.ast.DuckDBExpression; +import sqlancer.duckdb.ast.DuckDBFunction; import sqlancer.duckdb.ast.DuckDBSelect; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBAggregateFunction; import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBBinaryArithmeticOperator; @@ -49,9 +47,8 @@ public void check() throws SQLException { DuckDBAggregateFunction aggregateFunction = Randomly.fromOptions(DuckDBAggregateFunction.MAX, DuckDBAggregateFunction.MIN, DuckDBAggregateFunction.SUM, DuckDBAggregateFunction.COUNT, DuckDBAggregateFunction.AVG/* , DuckDBAggregateFunction.STDDEV_POP */); - NewFunctionNode aggregate = gen - .generateArgsForAggregate(aggregateFunction); - List> fetchColumns = new ArrayList<>(); + DuckDBFunction aggregate = gen.generateArgsForAggregate(aggregateFunction); + List fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { fetchColumns.add(gen.generateAggregate()); @@ -78,15 +75,15 @@ public void check() throws SQLException { } - private String createMetamorphicUnionQuery(DuckDBSelect select, - NewFunctionNode aggregate, List> from) { + private String createMetamorphicUnionQuery(DuckDBSelect select, DuckDBFunction aggregate, + List from) { String metamorphicQuery; - Node whereClause = gen.generateExpression(); - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + DuckDBExpression whereClause = gen.generateExpression(); + DuckDBExpression negatedClause = new sqlancer.duckdb.ast.DuckDBUnaryPrefixOperator(whereClause, DuckDBUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + DuckDBExpression notNullClause = new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(whereClause, DuckDBUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); + List mappedAggregate = mapped(aggregate); DuckDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DuckDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DuckDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); @@ -120,7 +117,7 @@ private String getAggregateResult(String queryString) throws SQLException { } } - private List> mapped(NewFunctionNode aggregate) { + private List mapped(DuckDBFunction aggregate) { DuckDBCastOperation count; switch (aggregate.getFunc()) { case COUNT: @@ -129,21 +126,19 @@ private List> mapped(NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + DuckDBFunction sum = new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.SUM); - count = new DuckDBCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DuckDBAggregateFunction.COUNT), + count = new DuckDBCastOperation(new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.COUNT), new DuckDBCompositeDataType(DuckDBDataType.FLOAT, 8)); return aliasArgs(Arrays.asList(sum, count)); case STDDEV_POP: - NewFunctionNode sumSquared = new NewFunctionNode<>( - Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DuckDBFunction sumSquared = new DuckDBFunction<>( + Arrays.asList(new DuckDBBinaryOperator(aggregate.getArgs().get(0), aggregate.getArgs().get(0), DuckDBBinaryArithmeticOperator.MULT)), DuckDBAggregateFunction.SUM); - count = new DuckDBCastOperation( - new NewFunctionNode(aggregate.getArgs(), - DuckDBAggregateFunction.COUNT), + count = new DuckDBCastOperation(new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.COUNT), new DuckDBCompositeDataType(DuckDBDataType.FLOAT, 8)); - NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), + DuckDBFunction avg = new DuckDBFunction<>(aggregate.getArgs(), DuckDBAggregateFunction.AVG); return aliasArgs(Arrays.asList(sumSquared, count, avg)); default: @@ -151,11 +146,11 @@ private List> mapped(NewFunctionNode> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode(expr, "agg" + i++)); + for (DuckDBExpression expr : originalAggregateArgs) { + args.add(new DuckDBAlias(expr, "agg" + i++)); } return args; } @@ -173,8 +168,8 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, - Node whereClause, List> joinList) { + private DuckDBSelect getSelect(List aggregates, List from, + DuckDBExpression whereClause, List joinList) { DuckDBSelect leftSelect = new DuckDBSelect(); leftSelect.setFetchColumns(aggregates); leftSelect.setFromList(from); diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index 2acc6a6d6..b4858df4c 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -6,9 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -18,13 +15,14 @@ import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBSchema.DuckDBTables; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -public class DuckDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, DuckDBGlobalState> +public class DuckDBQueryPartitioningBase extends TernaryLogicPartitioningOracleBase implements TestOracle { DuckDBSchema s; @@ -46,27 +44,27 @@ public void check() throws SQLException { select = new DuckDBSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DuckDBJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + List joins = DuckDBJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new DuckDBColumn("*", null, false, false))); + columns.add(new DuckDBColumnReference(new DuckDBColumn("*", null, false, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); } return columns; } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java index 86c26370d..c40a63a18 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningGroupByTester.java @@ -7,12 +7,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; public class DuckDBQueryPartitioningGroupByTester extends DuckDBQueryPartitioningBase { @@ -45,9 +43,9 @@ public void check() throws SQLException { } @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); } } diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java index b75bf02a5..b0ff0a44c 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningHavingTester.java @@ -7,7 +7,6 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -51,12 +50,12 @@ public void check() throws SQLException { } @Override - protected Node generatePredicate() { + protected DuckDBExpression generatePredicate() { return gen.generateHavingClause(); } @Override - List> generateFetchColumns() { + List generateFetchColumns() { return Arrays.asList(gen.generateHavingClause()); } From eac479e2d61c5c8bc817c25e3b434e92304c153d Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 23 Aug 2024 09:54:57 +0800 Subject: [PATCH 470/856] Remove Node supertype for new AST expressions --- src/sqlancer/common/ast/newast/ColumnReferenceNode.java | 2 +- src/sqlancer/common/ast/newast/Join.java | 2 -- src/sqlancer/common/ast/newast/NewAliasNode.java | 2 +- src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewCaseOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewFunctionNode.java | 2 +- src/sqlancer/common/ast/newast/NewInOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewOrderingTerm.java | 2 +- src/sqlancer/common/ast/newast/NewPostfixTextNode.java | 2 +- src/sqlancer/common/ast/newast/NewTernaryNode.java | 2 +- src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java | 2 +- src/sqlancer/common/ast/newast/TableReferenceNode.java | 2 +- 14 files changed, 13 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/common/ast/newast/ColumnReferenceNode.java b/src/sqlancer/common/ast/newast/ColumnReferenceNode.java index 18ff09db6..2d8688d8f 100644 --- a/src/sqlancer/common/ast/newast/ColumnReferenceNode.java +++ b/src/sqlancer/common/ast/newast/ColumnReferenceNode.java @@ -2,7 +2,7 @@ import sqlancer.common.schema.AbstractTableColumn; -public class ColumnReferenceNode> implements Node { +public class ColumnReferenceNode> { private final C c; diff --git a/src/sqlancer/common/ast/newast/Join.java b/src/sqlancer/common/ast/newast/Join.java index 90935b338..94d26c139 100644 --- a/src/sqlancer/common/ast/newast/Join.java +++ b/src/sqlancer/common/ast/newast/Join.java @@ -6,7 +6,5 @@ public interface Join, T extends AbstractTable, C extends AbstractTableColumn> extends Expression { - Expression getOnClause(); - void setOnClause(E onClause); } diff --git a/src/sqlancer/common/ast/newast/NewAliasNode.java b/src/sqlancer/common/ast/newast/NewAliasNode.java index 9fbbb35d5..260900712 100644 --- a/src/sqlancer/common/ast/newast/NewAliasNode.java +++ b/src/sqlancer/common/ast/newast/NewAliasNode.java @@ -1,6 +1,6 @@ package sqlancer.common.ast.newast; -public class NewAliasNode implements Node { +public class NewAliasNode { private final E expr; private final String alias; diff --git a/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java index e47012267..4bb9ce17a 100644 --- a/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBetweenOperatorNode.java @@ -1,6 +1,6 @@ package sqlancer.common.ast.newast; -public class NewBetweenOperatorNode implements Node { +public class NewBetweenOperatorNode { protected T left; protected T middle; diff --git a/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java index 6f150499c..b2fcc5ef0 100644 --- a/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewBinaryOperatorNode.java @@ -2,7 +2,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; -public class NewBinaryOperatorNode implements Node { +public class NewBinaryOperatorNode { protected final Operator op; protected final T left; diff --git a/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java index 5ab2ed853..440078566 100644 --- a/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewCaseOperatorNode.java @@ -2,7 +2,7 @@ import java.util.List; -public class NewCaseOperatorNode implements Node { +public class NewCaseOperatorNode { private final List conditions; private final List expressions; diff --git a/src/sqlancer/common/ast/newast/NewFunctionNode.java b/src/sqlancer/common/ast/newast/NewFunctionNode.java index baa5d1bc3..cdd91bd61 100644 --- a/src/sqlancer/common/ast/newast/NewFunctionNode.java +++ b/src/sqlancer/common/ast/newast/NewFunctionNode.java @@ -2,7 +2,7 @@ import java.util.List; -public class NewFunctionNode implements Node { +public class NewFunctionNode { protected List args; protected F func; diff --git a/src/sqlancer/common/ast/newast/NewInOperatorNode.java b/src/sqlancer/common/ast/newast/NewInOperatorNode.java index b4bcf535d..94a3a0886 100644 --- a/src/sqlancer/common/ast/newast/NewInOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewInOperatorNode.java @@ -2,7 +2,7 @@ import java.util.List; -public class NewInOperatorNode implements Node { +public class NewInOperatorNode { private final T left; private final List right; diff --git a/src/sqlancer/common/ast/newast/NewOrderingTerm.java b/src/sqlancer/common/ast/newast/NewOrderingTerm.java index be938fbb4..2efed05bc 100644 --- a/src/sqlancer/common/ast/newast/NewOrderingTerm.java +++ b/src/sqlancer/common/ast/newast/NewOrderingTerm.java @@ -2,7 +2,7 @@ import sqlancer.Randomly; -public class NewOrderingTerm implements Node { +public class NewOrderingTerm { private final T expr; private final Ordering ordering; diff --git a/src/sqlancer/common/ast/newast/NewPostfixTextNode.java b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java index 4d03a300c..716dbf8e3 100644 --- a/src/sqlancer/common/ast/newast/NewPostfixTextNode.java +++ b/src/sqlancer/common/ast/newast/NewPostfixTextNode.java @@ -1,6 +1,6 @@ package sqlancer.common.ast.newast; -public class NewPostfixTextNode implements Node { +public class NewPostfixTextNode { private final T expr; private final String text; diff --git a/src/sqlancer/common/ast/newast/NewTernaryNode.java b/src/sqlancer/common/ast/newast/NewTernaryNode.java index ab884e48c..2ca7b1f30 100644 --- a/src/sqlancer/common/ast/newast/NewTernaryNode.java +++ b/src/sqlancer/common/ast/newast/NewTernaryNode.java @@ -1,6 +1,6 @@ package sqlancer.common.ast.newast; -public class NewTernaryNode implements Node { +public class NewTernaryNode { protected final T left; protected final T middle; diff --git a/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java index fb7c28a0f..b3ccd4cd3 100644 --- a/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPostfixOperatorNode.java @@ -2,7 +2,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; -public class NewUnaryPostfixOperatorNode implements Node { +public class NewUnaryPostfixOperatorNode { protected final Operator op; private final T expr; diff --git a/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java index df75140f7..8668aec35 100644 --- a/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java +++ b/src/sqlancer/common/ast/newast/NewUnaryPrefixOperatorNode.java @@ -2,7 +2,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; -public class NewUnaryPrefixOperatorNode implements Node { +public class NewUnaryPrefixOperatorNode { protected final Operator op; private final T expr; diff --git a/src/sqlancer/common/ast/newast/TableReferenceNode.java b/src/sqlancer/common/ast/newast/TableReferenceNode.java index 981da70ad..0719a0ed6 100644 --- a/src/sqlancer/common/ast/newast/TableReferenceNode.java +++ b/src/sqlancer/common/ast/newast/TableReferenceNode.java @@ -2,7 +2,7 @@ import sqlancer.common.schema.AbstractTable; -public class TableReferenceNode> implements Node { +public class TableReferenceNode> { private final T t; From baa1f1c04e6d97bf12b33c83475e18ac3fb94544 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 23 Aug 2024 11:04:13 +0800 Subject: [PATCH 471/856] [Databend] Update expressions to match new AST --- .../DatabendExpectedValueVisitor.java | 103 +++++++++--------- src/sqlancer/databend/DatabendExprToNode.java | 25 ----- .../databend/DatabendToStringVisitor.java | 5 +- .../ast/DatabendAggregateOperation.java | 9 +- src/sqlancer/databend/ast/DatabendAlias.java | 9 ++ .../ast/DatabendBetweenOperation.java | 9 +- .../DatabendBinaryArithmeticOperation.java | 7 +- .../DatabendBinaryComparisonOperation.java | 7 +- .../ast/DatabendBinaryLogicalOperation.java | 7 +- .../databend/ast/DatabendBinaryOperation.java | 12 ++ .../databend/ast/DatabendCastOperation.java | 3 +- .../databend/ast/DatabendColumnReference.java | 11 ++ .../databend/ast/DatabendConstant.java | 3 +- .../ast/DatabendFunctionOperation.java | 9 +- .../databend/ast/DatabendInOperation.java | 3 +- src/sqlancer/databend/ast/DatabendJoin.java | 49 ++++----- .../databend/ast/DatabendLikeOperation.java | 7 +- .../databend/ast/DatabendOrderByTerm.java | 3 +- .../databend/ast/DatabendPostFixText.java | 9 ++ src/sqlancer/databend/ast/DatabendSelect.java | 3 +- .../databend/ast/DatabendTableReference.java | 11 ++ .../ast/DatabendUnaryPostfixOperation.java | 3 +- .../ast/DatabendUnaryPrefixOperation.java | 3 +- .../databend/gen/DatabendDeleteGenerator.java | 5 +- .../databend/gen/DatabendInsertGenerator.java | 6 +- .../gen/DatabendNewExpressionGenerator.java | 3 +- .../gen/DatabendRandomQuerySynthesizer.java | 23 ++-- .../databend/gen/DatabendTableGenerator.java | 4 +- .../databend/test/DatabendNoRECOracle.java | 37 +++---- .../DatabendPivotedQuerySynthesisOracle.java | 28 ++--- ...abendQueryPartitioningAggregateTester.java | 62 +++++------ .../tlp/DatabendQueryPartitioningBase.java | 31 +++--- ...tabendQueryPartitioningDistinctTester.java | 7 +- ...atabendQueryPartitioningGroupByTester.java | 17 ++- ...DatabendQueryPartitioningHavingTester.java | 17 ++- .../DatabendQueryPartitioningWhereTester.java | 7 +- .../doris/visitor/DorisExprToNode.java | 25 ----- 37 files changed, 268 insertions(+), 314 deletions(-) delete mode 100644 src/sqlancer/databend/DatabendExprToNode.java create mode 100644 src/sqlancer/databend/ast/DatabendAlias.java create mode 100644 src/sqlancer/databend/ast/DatabendBinaryOperation.java create mode 100644 src/sqlancer/databend/ast/DatabendColumnReference.java create mode 100644 src/sqlancer/databend/ast/DatabendPostFixText.java create mode 100644 src/sqlancer/databend/ast/DatabendTableReference.java delete mode 100644 src/sqlancer/doris/visitor/DorisExprToNode.java diff --git a/src/sqlancer/databend/DatabendExpectedValueVisitor.java b/src/sqlancer/databend/DatabendExpectedValueVisitor.java index f6ff93819..e10ade97a 100644 --- a/src/sqlancer/databend/DatabendExpectedValueVisitor.java +++ b/src/sqlancer/databend/DatabendExpectedValueVisitor.java @@ -2,60 +2,57 @@ import java.util.List; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.ast.DatabendAlias; +import sqlancer.databend.ast.DatabendBetweenOperation; +import sqlancer.databend.ast.DatabendBinaryOperation; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendFunctionOperation; +import sqlancer.databend.ast.DatabendInOperation; import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendOrderByTerm; +import sqlancer.databend.ast.DatabendPostFixText; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; public class DatabendExpectedValueVisitor { protected final StringBuilder sb = new StringBuilder(); - private void print(Node expr) { + private void print(DatabendExpression expr) { sb.append(DatabendToStringVisitor.asString(expr)); sb.append(" -- "); - sb.append(((DatabendExpression) expr).getExpectedValue()); + sb.append((expr).getExpectedValue()); sb.append("\n"); } - @SuppressWarnings("unchecked") - public void visit(Node expr) { + public void visit(DatabendExpression expr) { assert expr != null; - if (expr instanceof ColumnReferenceNode) { - visit((ColumnReferenceNode) expr); - } else if (expr instanceof NewUnaryPostfixOperatorNode) { - visit((NewUnaryPostfixOperatorNode) expr); - } else if (expr instanceof NewUnaryPrefixOperatorNode) { - visit((NewUnaryPrefixOperatorNode) expr); - } else if (expr instanceof NewBinaryOperatorNode) { - visit((NewBinaryOperatorNode) expr); - } else if (expr instanceof TableReferenceNode) { - visit((TableReferenceNode) expr); - } else if (expr instanceof NewFunctionNode) { - visit((NewFunctionNode) expr); - } else if (expr instanceof NewBetweenOperatorNode) { - visit((NewBetweenOperatorNode) expr); - } else if (expr instanceof NewInOperatorNode) { - visit((NewInOperatorNode) expr); - } else if (expr instanceof NewOrderingTerm) { - visit((NewOrderingTerm) expr); - } else if (expr instanceof NewAliasNode) { - visit((NewAliasNode) expr); - } else if (expr instanceof NewPostfixTextNode) { - visit((NewPostfixTextNode) expr); + if (expr instanceof DatabendColumnReference) { + visit((DatabendColumnReference) expr); + } else if (expr instanceof DatabendUnaryPostfixOperation) { + visit((DatabendUnaryPostfixOperation) expr); + } else if (expr instanceof DatabendUnaryPrefixOperation) { + visit((DatabendUnaryPrefixOperation) expr); + } else if (expr instanceof DatabendBinaryOperation) { + visit((DatabendBinaryOperation) expr); + } else if (expr instanceof DatabendTableReference) { + visit((DatabendTableReference) expr); + } else if (expr instanceof DatabendFunctionOperation) { + visit((DatabendFunctionOperation) expr); + } else if (expr instanceof DatabendBetweenOperation) { + visit((DatabendBetweenOperation) expr); + } else if (expr instanceof DatabendInOperation) { + visit((DatabendInOperation) expr); + } else if (expr instanceof DatabendOrderByTerm) { + visit((DatabendOrderByTerm) expr); + } else if (expr instanceof DatabendAlias) { + visit((DatabendAlias) expr); + } else if (expr instanceof DatabendPostFixText) { + visit((DatabendPostFixText) expr); } else if (expr instanceof DatabendConstant) { visit((DatabendConstant) expr); } else if (expr instanceof DatabendSelect) { @@ -67,65 +64,65 @@ public void visit(Node expr) { } } - public void visit(ColumnReferenceNode c) { + public void visit(DatabendColumnReference c) { print(c); } - public void visit(NewUnaryPostfixOperatorNode op) { + public void visit(DatabendUnaryPostfixOperation op) { print(op); visit(op.getExpr()); } - public void visit(NewUnaryPrefixOperatorNode op) { + public void visit(DatabendUnaryPrefixOperation op) { print(op); visit(op.getExpr()); } - public void visit(NewBinaryOperatorNode op) { + public void visit(DatabendBinaryOperation op) { print(op); visit(op.getLeft()); visit(op.getRight()); } - public void visit(TableReferenceNode t) { + public void visit(DatabendTableReference t) { print(t); } - public void visit(NewFunctionNode fun) { + public void visit(DatabendFunctionOperation fun) { print(fun); visit(fun.getArgs()); } - public void visit(List> expressions) { - for (Node expression : expressions) { + public void visit(List expressions) { + for (DatabendExpression expression : expressions) { visit(expression); } } - public void visit(NewBetweenOperatorNode op) { + public void visit(DatabendBetweenOperation op) { print(op); visit(op.getLeft()); visit(op.getMiddle()); visit(op.getRight()); } - public void visit(NewInOperatorNode op) { + public void visit(DatabendInOperation op) { print(op); visit(op.getLeft()); visit(op.getRight()); } - public void visit(NewOrderingTerm op) { + public void visit(DatabendOrderByTerm op) { print(op); visit(op.getExpr()); } - public void visit(NewAliasNode op) { + public void visit(DatabendAlias op) { print(op); visit(op.getExpr()); } - public void visit(NewPostfixTextNode postFixText) { + public void visit(DatabendPostFixText postFixText) { print(postFixText); visit(postFixText.getExpr()); } @@ -146,7 +143,7 @@ public String get() { return sb.toString(); } - public static String asExpectedValues(Node expr) { + public static String asExpectedValues(DatabendExpression expr) { DatabendExpectedValueVisitor v = new DatabendExpectedValueVisitor(); v.visit(expr); return v.get(); diff --git a/src/sqlancer/databend/DatabendExprToNode.java b/src/sqlancer/databend/DatabendExprToNode.java deleted file mode 100644 index 0b2815f5c..000000000 --- a/src/sqlancer/databend/DatabendExprToNode.java +++ /dev/null @@ -1,25 +0,0 @@ -package sqlancer.databend; - -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.ast.newast.Node; -import sqlancer.databend.ast.DatabendExpression; - -public final class DatabendExprToNode { - - private DatabendExprToNode() { - - } - - @SuppressWarnings("unchecked") - public static Node cast(DatabendExpression expression) { - return (Node) expression; - } - - @SuppressWarnings("unchecked") - public static List> casts(List expressions) { - return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); - } - -} diff --git a/src/sqlancer/databend/DatabendToStringVisitor.java b/src/sqlancer/databend/DatabendToStringVisitor.java index 87c94d074..05d09d7f2 100644 --- a/src/sqlancer/databend/DatabendToStringVisitor.java +++ b/src/sqlancer/databend/DatabendToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.databend; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; @@ -10,7 +9,7 @@ public class DatabendToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(DatabendExpression expr) { if (expr instanceof DatabendConstant) { visit((DatabendConstant) expr); } else if (expr instanceof DatabendSelect) { @@ -91,7 +90,7 @@ private void visit(DatabendSelect select) { } } - public static String asString(Node expr) { + public static String asString(DatabendExpression expr) { DatabendToStringVisitor visitor = new DatabendToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/databend/ast/DatabendAggregateOperation.java b/src/sqlancer/databend/ast/DatabendAggregateOperation.java index df5e96199..5070991d8 100644 --- a/src/sqlancer/databend/ast/DatabendAggregateOperation.java +++ b/src/sqlancer/databend/ast/DatabendAggregateOperation.java @@ -3,15 +3,12 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema; -public class DatabendAggregateOperation - extends NewFunctionNode - implements DatabendExpression { +public class DatabendAggregateOperation extends + DatabendFunctionOperation implements DatabendExpression { public DatabendAggregateOperation(List args, DatabendAggregateFunction func) { - super(DatabendExprToNode.casts(args), func); + super(args, func); } public enum DatabendAggregateFunction { diff --git a/src/sqlancer/databend/ast/DatabendAlias.java b/src/sqlancer/databend/ast/DatabendAlias.java new file mode 100644 index 000000000..5f6827409 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendAlias.java @@ -0,0 +1,9 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class DatabendAlias extends NewAliasNode implements DatabendExpression { + public DatabendAlias(DatabendExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/databend/ast/DatabendBetweenOperation.java b/src/sqlancer/databend/ast/DatabendBetweenOperation.java index 27f5532c4..a6a0134bf 100644 --- a/src/sqlancer/databend/ast/DatabendBetweenOperation.java +++ b/src/sqlancer/databend/ast/DatabendBetweenOperation.java @@ -1,25 +1,24 @@ package sqlancer.databend.ast; import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema; public class DatabendBetweenOperation extends NewBetweenOperatorNode implements DatabendExpression { public DatabendBetweenOperation(DatabendExpression left, DatabendExpression middle, DatabendExpression right, boolean isTrue) { - super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(middle), DatabendExprToNode.cast(right), isTrue); + super(left, middle, right, isTrue); } public DatabendExpression getLeftExpr() { - return (DatabendExpression) left; + return left; } public DatabendExpression getMiddleExpr() { - return (DatabendExpression) middle; + return middle; } public DatabendExpression getRightExpr() { - return (DatabendExpression) right; + return right; } @Override diff --git a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java index 391597941..5554cc474 100644 --- a/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java +++ b/src/sqlancer/databend/ast/DatabendBinaryArithmeticOperation.java @@ -4,7 +4,6 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode @@ -12,7 +11,7 @@ public class DatabendBinaryArithmeticOperation extends NewBinaryOperatorNode @@ -10,15 +9,15 @@ public class DatabendBinaryComparisonOperation extends NewBinaryOperatorNode @@ -11,15 +10,15 @@ public class DatabendBinaryLogicalOperation extends NewBinaryOperatorNode implements DatabendExpression { + public DatabendBinaryOperation(DatabendExpression left, DatabendExpression right, + BinaryOperatorNode.Operator operator) { + super(left, right, operator); + } + +} diff --git a/src/sqlancer/databend/ast/DatabendCastOperation.java b/src/sqlancer/databend/ast/DatabendCastOperation.java index 534836d59..07e7d7f9c 100644 --- a/src/sqlancer/databend/ast/DatabendCastOperation.java +++ b/src/sqlancer/databend/ast/DatabendCastOperation.java @@ -2,7 +2,6 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; @@ -11,7 +10,7 @@ public class DatabendCastOperation extends NewUnaryPostfixOperatorNode expr, DatabendCompositeDataType type) { + public DatabendCastOperation(DatabendExpression expr, DatabendCompositeDataType type) { super(expr, new BinaryOperatorNode.Operator() { @Override public String getTextRepresentation() { diff --git a/src/sqlancer/databend/ast/DatabendColumnReference.java b/src/sqlancer/databend/ast/DatabendColumnReference.java new file mode 100644 index 000000000..4b53ef4df --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendColumnReference extends ColumnReferenceNode + implements DatabendExpression { + public DatabendColumnReference(DatabendSchema.DatabendColumn column) { + super(column); + } +} diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 0cc3d0bd2..435bf6ece 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -3,10 +3,9 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendSchema.DatabendDataType; -public abstract class DatabendConstant implements Node, DatabendExpression { +public abstract class DatabendConstant implements DatabendExpression { private DatabendConstant() { } diff --git a/src/sqlancer/databend/ast/DatabendFunctionOperation.java b/src/sqlancer/databend/ast/DatabendFunctionOperation.java index 855e1c8f1..69cb16175 100644 --- a/src/sqlancer/databend/ast/DatabendFunctionOperation.java +++ b/src/sqlancer/databend/ast/DatabendFunctionOperation.java @@ -1,4 +1,11 @@ package sqlancer.databend.ast; -public class DatabendFunctionOperation { // TODO add databend function +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DatabendFunctionOperation extends NewFunctionNode implements DatabendExpression { + public DatabendFunctionOperation(List args, F func) { + super(args, func); + } } diff --git a/src/sqlancer/databend/ast/DatabendInOperation.java b/src/sqlancer/databend/ast/DatabendInOperation.java index 304c3e3f9..f382aae04 100644 --- a/src/sqlancer/databend/ast/DatabendInOperation.java +++ b/src/sqlancer/databend/ast/DatabendInOperation.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema; public class DatabendInOperation extends NewInOperatorNode implements DatabendExpression { @@ -12,7 +11,7 @@ public class DatabendInOperation extends NewInOperatorNode i private final List rightExpr; public DatabendInOperation(DatabendExpression left, List right, boolean isNegated) { - super(DatabendExprToNode.cast(left), DatabendExprToNode.casts(right), isNegated); + super(left, right, isNegated); this.leftExpr = left; this.rightExpr = right; } diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index 123916542..b927eab92 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -4,21 +4,19 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendJoin implements Node { +public class DatabendJoin implements DatabendExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final DatabendTableReference leftTable; + private final DatabendTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final DatabendExpression onCondition; private OuterType outerType; public enum JoinType { @@ -37,9 +35,8 @@ public static OuterType getRandom() { } } - public DatabendJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public DatabendJoin(DatabendTableReference leftTable, DatabendTableReference rightTable, JoinType joinType, + DatabendExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; @@ -58,7 +55,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public DatabendExpression getOnCondition() { return onCondition; } @@ -70,12 +67,12 @@ public OuterType getOuterType() { return outerType; } - public static List> getJoins( - List> tableList, DatabendGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, + DatabendGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + DatabendTableReference leftTable = tableList.remove(0); + DatabendTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); DatabendNewExpressionGenerator joinGen = new DatabendNewExpressionGenerator(globalState) @@ -84,18 +81,18 @@ public static List> getJoins( switch (JoinType.getRandom()) { case INNER: joinExpressions.add(DatabendJoin.createInnerJoin(leftTable, rightTable, - DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case NATURAL: joinExpressions.add(DatabendJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); break; case LEFT: joinExpressions.add(DatabendJoin.createLeftOuterJoin(leftTable, rightTable, - DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; case RIGHT: joinExpressions.add(DatabendJoin.createRightOuterJoin(leftTable, rightTable, - DatabendExprToNode.cast(joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)))); + joinGen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); break; default: throw new AssertionError(); @@ -104,23 +101,23 @@ public static List> getJoins( return joinExpressions; } - public static DatabendJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DatabendJoin createRightOuterJoin(DatabendTableReference left, DatabendTableReference right, + DatabendExpression predicate) { return new DatabendJoin(left, right, JoinType.RIGHT, predicate); } - public static DatabendJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DatabendJoin createLeftOuterJoin(DatabendTableReference left, DatabendTableReference right, + DatabendExpression predicate) { return new DatabendJoin(left, right, JoinType.LEFT, predicate); } - public static DatabendJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DatabendJoin createInnerJoin(DatabendTableReference left, DatabendTableReference right, + DatabendExpression predicate) { return new DatabendJoin(left, right, JoinType.INNER, predicate); } - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, OuterType naturalJoinType) { + public static DatabendExpression createNaturalJoin(DatabendTableReference left, DatabendTableReference right, + OuterType naturalJoinType) { DatabendJoin join = new DatabendJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; diff --git a/src/sqlancer/databend/ast/DatabendLikeOperation.java b/src/sqlancer/databend/ast/DatabendLikeOperation.java index 36ac73603..f40d804bb 100644 --- a/src/sqlancer/databend/ast/DatabendLikeOperation.java +++ b/src/sqlancer/databend/ast/DatabendLikeOperation.java @@ -4,13 +4,12 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; public class DatabendLikeOperation extends NewBinaryOperatorNode implements DatabendExpression { public DatabendLikeOperation(DatabendExpression left, DatabendExpression right, DatabendLikeOperator op) { - super(DatabendExprToNode.cast(left), DatabendExprToNode.cast(right), op); + super(left, right, op); } @Override @@ -19,11 +18,11 @@ public DatabendDataType getExpectedType() { } public DatabendExpression getLeftExpr() { - return (DatabendExpression) super.getLeft(); + return super.getLeft(); } public DatabendExpression getRightExpr() { - return (DatabendExpression) super.getRight(); + return super.getRight(); } public DatabendLikeOperator getOp() { diff --git a/src/sqlancer/databend/ast/DatabendOrderByTerm.java b/src/sqlancer/databend/ast/DatabendOrderByTerm.java index 5c78bae3f..7492c7c33 100644 --- a/src/sqlancer/databend/ast/DatabendOrderByTerm.java +++ b/src/sqlancer/databend/ast/DatabendOrderByTerm.java @@ -1,10 +1,9 @@ package sqlancer.databend.ast; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.Node; public class DatabendOrderByTerm extends NewOrderingTerm implements DatabendExpression { - public DatabendOrderByTerm(Node expr, Ordering ordering) { + public DatabendOrderByTerm(DatabendExpression expr, Ordering ordering) { super(expr, ordering); } } diff --git a/src/sqlancer/databend/ast/DatabendPostFixText.java b/src/sqlancer/databend/ast/DatabendPostFixText.java new file mode 100644 index 000000000..8f854fbe1 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendPostFixText.java @@ -0,0 +1,9 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class DatabendPostFixText extends NewPostfixTextNode implements DatabendExpression { + public DatabendPostFixText(DatabendExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/databend/ast/DatabendSelect.java b/src/sqlancer/databend/ast/DatabendSelect.java index b3ccc08f5..3b0854ede 100644 --- a/src/sqlancer/databend/ast/DatabendSelect.java +++ b/src/sqlancer/databend/ast/DatabendSelect.java @@ -1,9 +1,8 @@ package sqlancer.databend.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class DatabendSelect extends SelectBase> implements Node { +public class DatabendSelect extends SelectBase implements DatabendExpression { private boolean isDistinct; diff --git a/src/sqlancer/databend/ast/DatabendTableReference.java b/src/sqlancer/databend/ast/DatabendTableReference.java new file mode 100644 index 000000000..7a98d3877 --- /dev/null +++ b/src/sqlancer/databend/ast/DatabendTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.databend.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.databend.DatabendSchema; + +public class DatabendTableReference extends TableReferenceNode + implements DatabendExpression { + public DatabendTableReference(DatabendSchema.DatabendTable table) { + super(table); + } +} diff --git a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java index a19512edf..119f93e5c 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPostfixOperation.java @@ -3,14 +3,13 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; public class DatabendUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements DatabendExpression { public DatabendUnaryPostfixOperation(DatabendExpression expr, DatabendUnaryPostfixOperator op) { - super(DatabendExprToNode.cast(expr), op); + super(expr, op); } public DatabendExpression getExpression() { diff --git a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java index f115684e0..ea6b65f07 100644 --- a/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java +++ b/src/sqlancer/databend/ast/DatabendUnaryPrefixOperation.java @@ -3,14 +3,13 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendSchema.DatabendDataType; public class DatabendUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements DatabendExpression { public DatabendUnaryPrefixOperation(DatabendExpression expr, DatabendUnaryPrefixOperator op) { - super(DatabendExprToNode.cast(expr), op); + super(expr, op); } public DatabendExpression getExpression() { diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index 9fabfc125..256030409 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -4,7 +4,6 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; @@ -20,8 +19,8 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); - sb.append(DatabendToStringVisitor.asString(DatabendExprToNode.cast( - new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN)))); + sb.append(DatabendToStringVisitor.asString( + new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN))); DatabendErrors.addExpressionErrors(errors); } return new SQLQueryAdapter(sb.toString(), errors); diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index a86f0f3a9..340e494ef 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -8,7 +8,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; @@ -49,9 +48,8 @@ protected void insertValue(DatabendColumn column) { // } else { // sb.append(DatabendToStringVisitor.asString(new DatabendExpressionGenerator(globalState).generateConstant())); // } - String value = DatabendToStringVisitor - .asString(DatabendExprToNode.cast(new DatabendNewExpressionGenerator(globalState) - .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); // 生成一个与column相同的常量类型 + String value = DatabendToStringVisitor.asString(new DatabendNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 sb.append(value); } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 2faf4376b..19a45eaa2 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -9,7 +9,6 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendProvider.DatabendGlobalState; @@ -93,7 +92,7 @@ private DatabendExpression createColumnOfType(DatabendDataType type) { return DatabendColumnValue.create(column, value); } - public List> generateOrderBy() { + public List generateOrderBy() { List randomColumns = Randomly.subset(columns); return randomColumns.stream().map( c -> new DatabendOrderByTerm(new DatabendColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 4f0b2fd00..7b7d0e35b 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -6,9 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -19,6 +16,7 @@ import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; public final class DatabendRandomQuerySynthesizer { @@ -30,19 +28,19 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int List targetColumns = targetTables.getColumns(); DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(targetColumns); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); HashSet columnOfLeafNode = new HashSet<>(); gen.setColumnOfLeafNode(columnOfLeafNode); int freeColumns = targetColumns.size(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node column = null; + DatabendExpression column = null; if (freeColumns > 0 && Randomly.getBoolean()) { column = new DatabendColumnValue(targetColumns.get(freeColumns - 1), null); freeColumns -= 1; columnOfLeafNode.add((DatabendColumnValue) column); } else { - column = DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); + column = gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN); } columns.add(column); } @@ -51,17 +49,16 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int select.setDistinct(isDistinct); select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DatabendJoin.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); + List joins = DatabendJoin.getJoins(tableList, globalState); select.setJoinList(joins); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { - select.setWhereClause( - DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } - List> noExprColumns = new ArrayList<>(columnOfLeafNode); + List noExprColumns = new ArrayList<>(columnOfLeafNode); if (Randomly.getBoolean() && !noExprColumns.isEmpty() && !isDistinct) { select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); @@ -71,7 +68,7 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int } if (Randomly.getBoolean()) { // 可能产生新的column叶子结点 - select.setHavingClause(DatabendExprToNode.cast(gen.generateHavingClause())); + select.setHavingClause(gen.generateHavingClause()); } noExprColumns = new ArrayList<>(columnOfLeafNode); diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index ab4900c07..514d740d0 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -8,7 +8,6 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -47,8 +46,7 @@ public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { sb.append(" DEFAULT("); sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 - DatabendExprToNode - .cast(gen.generateConstant(columns.get(i).getType().getPrimitiveDataType())))); + gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); sb.append(")"); } } diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 50a8c4de4..54f17858c 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -9,16 +9,11 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -28,9 +23,12 @@ import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendCastOperation; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendPostFixText; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.gen.DatabendNewExpressionGenerator; public class DatabendNoRECOracle extends NoRECBase implements TestOracle { @@ -51,12 +49,11 @@ public void check() throws SQLException { debugColumns(columns, randomTables); // 调试代码，可忽略 } DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = DatabendExprToNode - .cast(gen.generateExpression(DatabendDataType.BOOLEAN)); // 生成随机where条件 + DatabendExpression randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件 List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DatabendJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); + List joins = DatabendJoin.getJoins(tableList, state); int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, @@ -70,12 +67,12 @@ public void check() throws SQLException { } } - private int getUnoptimizedQueryCount(List> tableList, - Node randomWhereCondition, List> joins) throws SQLException { + private int getUnoptimizedQueryCount(List tableList, DatabendExpression randomWhereCondition, + List joins) throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); - Node asText = new NewPostfixTextNode<>(new DatabendCastOperation( - new NewPostfixTextNode(randomWhereCondition, + DatabendExpression asText = new DatabendPostFixText(new DatabendCastOperation( + new DatabendPostFixText(randomWhereCondition, " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); @@ -101,20 +98,18 @@ private int getUnoptimizedQueryCount(List> tableList, return secondCount; } - private int getOptimizedQueryCount(SQLConnection con, List> tableList, - List columns, Node randomWhereCondition, - List> joins) throws SQLException { + private int getOptimizedQueryCount(SQLConnection con, List tableList, + List columns, DatabendExpression randomWhereCondition, List joins) + throws SQLException { DatabendSelect select = new DatabendSelect(); // select.setGroupByClause(groupBys); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)) + List allColumns = columns.stream().map((c) -> new DatabendColumnReference(c)) .collect(Collectors.toList()); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new DatabendNewExpressionGenerator(state).setColumns(columns).generateOrderBys() - .stream().map(DatabendExprToNode::cast).collect(Collectors.toList())); + select.setOrderByClauses(new DatabendNewExpressionGenerator(state).setColumns(columns).generateOrderBys()); } select.setJoinList(joins); int firstCount = 0; diff --git a/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java index bfb21414e..4a72454b5 100644 --- a/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/databend/test/DatabendPivotedQuerySynthesisOracle.java @@ -6,25 +6,22 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendExpectedValueVisitor; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendSchema.DatabendRowValue; -import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.gen.DatabendNewExpressionGenerator; @@ -52,21 +49,21 @@ protected Query getRectifiedQuery() throws Exception { selectStatement.setFetchColumns(fetchColumns.stream() .map(c -> new DatabendColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) .collect(Collectors.toList())); - selectStatement.setFromList(randomTables.getTables().stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList())); + selectStatement.setFromList( + randomTables.getTables().stream().map(t -> new DatabendTableReference(t)).collect(Collectors.toList())); DatabendExpression whereClause = generateRectifiedExpression(columns, pivotRow); - selectStatement.setWhereClause(DatabendExprToNode.cast(whereClause)); - List> groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); - Node limitClause = generateLimit(); + DatabendExpression limitClause = generateLimit(); selectStatement.setLimitClause(limitClause); if (limitClause != null) { - Node offsetClause = generateOffset(); + DatabendExpression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); if (!isDistinct) { - List> orderBys = gen.generateOrderBy(); + List orderBys = gen.generateOrderBy(); selectStatement.setOrderByClauses(orderBys); } return new SQLQueryAdapter(DatabendToStringVisitor.asString(selectStatement), errors); @@ -122,11 +119,10 @@ private DatabendColumn getFetchValueAliasedColumn(DatabendColumn c) { @Override protected String getExpectedValues(DatabendExpression expr) { - return DatabendExpectedValueVisitor.asExpectedValues(DatabendExprToNode.cast(expr)); + return DatabendExpectedValueVisitor.asExpectedValues(expr); } - private List> generateGroupByClause(List columns, - DatabendRowValue rowValue) { + private List generateGroupByClause(List columns, DatabendRowValue rowValue) { if (Randomly.getBoolean()) { return columns.stream().map(c -> new DatabendColumnValue(c, rowValue.getValues().get(c))) .collect(Collectors.toList()); @@ -135,7 +131,7 @@ private List> generateGroupByClause(List generateLimit() { + private DatabendExpression generateLimit() { if (Randomly.getBoolean()) { return DatabendConstant.createIntConstant(Integer.MAX_VALUE); } else { @@ -143,7 +139,7 @@ private Node generateLimit() { } } - private Node generateOffset() { + private DatabendExpression generateOffset() { if (Randomly.getBoolean()) { return DatabendConstant.createIntConstant(0); } else { diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 043e93963..0ba0d9c86 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -8,27 +8,25 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendAggregateOperation; import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; +import sqlancer.databend.ast.DatabendAlias; import sqlancer.databend.ast.DatabendBinaryArithmeticOperation.DatabendBinaryArithmeticOperator; +import sqlancer.databend.ast.DatabendBinaryOperation; import sqlancer.databend.ast.DatabendCastOperation; import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendFunctionOperation; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; +import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; public class DatabendQueryPartitioningAggregateTester extends DatabendQueryPartitioningBase { @@ -49,9 +47,9 @@ public void check() throws SQLException { DatabendAggregateFunction aggregateFunction = Randomly.fromOptions(DatabendAggregateFunction.MAX, DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT, DatabendAggregateFunction.AVG/* , DatabendAggregateFunction.STDDEV_POP */); - NewFunctionNode aggregate = (DatabendAggregateOperation) gen + DatabendFunctionOperation aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); - List> fetchColumns = new ArrayList<>(); + List fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { fetchColumns.add((DatabendAggregateOperation) gen.generateAggregate()); // TODO 更换成非聚合函数 @@ -79,16 +77,14 @@ public void check() throws SQLException { } private String createMetamorphicUnionQuery(DatabendSelect select, - NewFunctionNode aggregate, - List> from) { + DatabendFunctionOperation aggregate, List from) { String metamorphicQuery; - Node whereClause = DatabendExprToNode - .cast(gen.generateExpression(DatabendDataType.BOOLEAN)); - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + DatabendExpression whereClause = gen.generateExpression(DatabendDataType.BOOLEAN); + DatabendExpression negatedClause = new DatabendUnaryPrefixOperation(whereClause, DatabendUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + DatabendExpression notNullClause = new DatabendUnaryPostfixOperation(whereClause, DatabendUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); + List mappedAggregate = mapped(aggregate); DatabendSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DatabendSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DatabendSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); @@ -126,8 +122,7 @@ private String getAggregateResult(String queryString) throws SQLException { } } - private List> mapped( - NewFunctionNode aggregate) { + private List mapped(DatabendFunctionOperation aggregate) { DatabendCastOperation count; switch (aggregate.getFunc()) { case COUNT: @@ -136,22 +131,21 @@ private List> mapped( case SUM: return aliasArgs(Arrays.asList(aggregate)); case AVG: - NewFunctionNode sum = new NewFunctionNode<>( + DatabendFunctionOperation sum = new DatabendFunctionOperation<>( aggregate.getArgs(), DatabendAggregateFunction.SUM); count = new DatabendCastOperation( - new NewFunctionNode<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), + new DatabendFunctionOperation<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); return aliasArgs(Arrays.asList(sum, count)); case STDDEV_POP: - NewFunctionNode sumSquared = new NewFunctionNode<>( - Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DatabendFunctionOperation sumSquared = new DatabendFunctionOperation<>( + Arrays.asList(new DatabendBinaryOperation(aggregate.getArgs().get(0), aggregate.getArgs().get(0), DatabendBinaryArithmeticOperator.MULTIPLICATION)), DatabendAggregateFunction.SUM); count = new DatabendCastOperation( - new NewFunctionNode(aggregate.getArgs(), - DatabendAggregateFunction.COUNT), + new DatabendFunctionOperation<>(aggregate.getArgs(), DatabendAggregateFunction.COUNT), new DatabendCompositeDataType(DatabendDataType.FLOAT, 8)); - NewFunctionNode avg = new NewFunctionNode<>( + DatabendFunctionOperation avg = new DatabendFunctionOperation<>( aggregate.getArgs(), DatabendAggregateFunction.AVG); return aliasArgs(Arrays.asList(sumSquared, count, avg)); default: @@ -159,16 +153,16 @@ private List> mapped( } } - private List> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode(expr, "agg" + i++)); + for (DatabendExpression expr : originalAggregateArgs) { + args.add(new DatabendAlias(expr, "agg" + i++)); } return args; } - private String getOuterAggregateFunction(NewFunctionNode aggregate) { + private String getOuterAggregateFunction(DatabendFunctionOperation aggregate) { switch (aggregate.getFunc()) { case STDDEV_POP: return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; @@ -181,16 +175,16 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, - Node whereClause, List> joinList) { + private DatabendSelect getSelect(List aggregates, List from, + DatabendExpression whereClause, List joinList) { DatabendSelect select = new DatabendSelect(); select.setFetchColumns(aggregates); select.setFromList(from); select.setWhereClause(whereClause); select.setJoinList(joinList); if (Randomly.getBooleanWithSmallProbability()) { - select.setGroupByExpressions(List.of(DatabendExprToNode.cast(gen.generateConstant(DatabendDataType.INT)))); // TODO - // 仍可加强 + select.setGroupByExpressions(List.of(gen.generateConstant(DatabendDataType.INT))); // TODO + // 仍可加强 } return select; } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 0042f83d6..50a02f0ff 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -7,9 +7,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -20,10 +17,12 @@ import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.DatabendSchema.DatabendTables; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.gen.DatabendNewExpressionGenerator; public class DatabendQueryPartitioningBase @@ -35,7 +34,7 @@ public class DatabendQueryPartitioningBase DatabendNewExpressionGenerator gen; DatabendSelect select; - List> groupByExpression; + List groupByExpression; public DatabendQueryPartitioningBase(DatabendGlobalState state) { super(state); @@ -57,33 +56,33 @@ public void check() throws SQLException { .addAll(randomColumn.stream().map(c -> new DatabendColumnValue(c, null)).collect(Collectors.toList())); groupByExpression = new ArrayList<>(columnOfLeafNode); - select.setFetchColumns(randomColumn.stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList())); + select.setFetchColumns( + randomColumn.stream().map(c -> new DatabendColumnReference(c)).collect(Collectors.toList())); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); if (!DatabendBugs.bug9236) { - List> joins = DatabendJoin.getJoins(tableList, state); + List joins = DatabendJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); } select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new DatabendColumn("*", null, false, false))); + columns.add(new DatabendColumnReference(new DatabendColumn("*", null, false, false))); } else { columns = generateRandomColumns(); } return columns; } - List> generateRandomColumns() { - List> columns; - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateRandomColumns() { + List columns; + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DatabendColumnReference(c)) + .collect(Collectors.toList()); return columns; } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java index 430c5cfc6..ce20ef214 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningDistinctTester.java @@ -7,7 +7,6 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendToStringVisitor; @@ -31,11 +30,11 @@ public void check() throws SQLException { if (Randomly.getBoolean()) { select.setDistinct(false); } - select.setWhereClause(DatabendExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java index 6ec148ae8..9d45291b3 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningGroupByTester.java @@ -7,13 +7,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendExpression; public class DatabendQueryPartitioningGroupByTester extends DatabendQueryPartitioningBase { @@ -32,11 +29,11 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setWhereClause(DatabendExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, @@ -46,9 +43,9 @@ public void check() throws SQLException { } @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DatabendColumnReference(c)) + .collect(Collectors.toList()); } } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java index dfde7755f..33de8c896 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningHavingTester.java @@ -7,9 +7,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendToStringVisitor; @@ -27,14 +25,13 @@ public DatabendQueryPartitioningHavingTester(DatabendGlobalState state) { public void check() throws SQLException { super.check(); if (Randomly.getBoolean()) { - select.setWhereClause( - DatabendExprToNode.cast(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN))); + select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); } // boolean orderBy = Randomly.getBoolean(); boolean orderBy = false; // 关闭order by if (orderBy) { // TODO 生成columns.size()的子集，有个错误：order by 后不能直接union，需要包装一层select // select.setOrderByClauses(gen.generateOrderBys()); - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add(new DatabendConstant.DatabendIntConstant( Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); @@ -44,11 +41,11 @@ public void check() throws SQLException { String originalQueryString = DatabendToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setHavingClause(DatabendExprToNode.cast(predicate)); + select.setHavingClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setHavingClause(DatabendExprToNode.cast(negatedPredicate)); + select.setHavingClause(negatedPredicate); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setHavingClause(DatabendExprToNode.cast(isNullPredicate)); + select.setHavingClause(isNullPredicate); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, @@ -63,8 +60,8 @@ protected DatabendExpression generatePredicate() { } @Override - List> generateFetchColumns() { - return Collections.singletonList(DatabendExprToNode.cast(gen.generateHavingClause())); + List generateFetchColumns() { + return Collections.singletonList(gen.generateHavingClause()); } } diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index d6fb7f0e6..6ea6f5e57 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -6,7 +6,6 @@ import sqlancer.ComparatorHelper; import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendExprToNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendToStringVisitor; @@ -30,11 +29,11 @@ public void check() throws SQLException { // if (orderBy) { //TODO 待开启 // select.setOrderByClauses(gen.generateOrderBys()); // } - select.setWhereClause(DatabendExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(DatabendExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DatabendToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, diff --git a/src/sqlancer/doris/visitor/DorisExprToNode.java b/src/sqlancer/doris/visitor/DorisExprToNode.java deleted file mode 100644 index cec7aa061..000000000 --- a/src/sqlancer/doris/visitor/DorisExprToNode.java +++ /dev/null @@ -1,25 +0,0 @@ -package sqlancer.doris.visitor; - -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.ast.newast.Node; -import sqlancer.doris.ast.DorisExpression; - -public final class DorisExprToNode { - - private DorisExprToNode() { - - } - - @SuppressWarnings("unchecked") - public static Node cast(DorisExpression expression) { - return (Node) expression; - } - - @SuppressWarnings("unchecked") - public static List> casts(List expressions) { - return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); - } - -} From ecd3d35ab577cf79824de903d053852e5c234ef5 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 26 Aug 2024 17:57:46 +0800 Subject: [PATCH 472/856] [Doris] Update expressions to match new AST --- .../doris/ast/DorisAggregateOperation.java | 8 +- src/sqlancer/doris/ast/DorisAlias.java | 9 ++ .../doris/ast/DorisBetweenOperation.java | 9 +- .../ast/DorisBinaryArithmeticOperation.java | 7 +- .../ast/DorisBinaryComparisonOperation.java | 7 +- .../ast/DorisBinaryLogicalOperation.java | 7 +- .../doris/ast/DorisBinaryOperation.java | 10 ++ .../doris/ast/DorisCaseOperation.java | 4 +- .../doris/ast/DorisCastOperation.java | 13 ++- .../doris/ast/DorisColumnReference.java | 11 ++ src/sqlancer/doris/ast/DorisConstant.java | 3 +- src/sqlancer/doris/ast/DorisFunction.java | 11 ++ .../doris/ast/DorisFunctionOperation.java | 3 +- src/sqlancer/doris/ast/DorisInOperation.java | 3 +- src/sqlancer/doris/ast/DorisJoin.java | 56 +++++----- .../doris/ast/DorisLikeOperation.java | 7 +- src/sqlancer/doris/ast/DorisOrderByTerm.java | 3 +- src/sqlancer/doris/ast/DorisPostfixText.java | 9 ++ src/sqlancer/doris/ast/DorisSelect.java | 3 +- .../doris/ast/DorisTableReference.java | 11 ++ .../doris/ast/DorisUnaryPostfixOperation.java | 5 +- .../doris/ast/DorisUnaryPrefixOperation.java | 5 +- .../doris/gen/DorisInsertGenerator.java | 6 +- .../gen/DorisNewExpressionGenerator.java | 6 +- .../gen/DorisRandomQuerySynthesizer.java | 22 ++-- .../doris/gen/DorisTableGenerator.java | 6 +- .../doris/gen/DorisUpdateGenerator.java | 3 +- .../doris/oracle/DorisNoRECOracle.java | 36 +++---- .../DorisPivotedQuerySynthesisOracle.java | 27 +++-- ...DorisQueryPartitioningAggregateTester.java | 62 +++++------ .../tlp/DorisQueryPartitioningBase.java | 23 ++-- .../DorisQueryPartitioningDistinctTester.java | 7 +- .../DorisQueryPartitioningGroupByTester.java | 17 ++- .../DorisQueryPartitioningHavingTester.java | 17 ++- .../DorisQueryPartitioningWhereTester.java | 10 +- .../visitor/DorisExpectedValueVisitor.java | 102 +++++++++--------- .../doris/visitor/DorisToStringVisitor.java | 25 ++--- 37 files changed, 284 insertions(+), 289 deletions(-) create mode 100644 src/sqlancer/doris/ast/DorisAlias.java create mode 100644 src/sqlancer/doris/ast/DorisBinaryOperation.java create mode 100644 src/sqlancer/doris/ast/DorisColumnReference.java create mode 100644 src/sqlancer/doris/ast/DorisFunction.java create mode 100644 src/sqlancer/doris/ast/DorisPostfixText.java create mode 100644 src/sqlancer/doris/ast/DorisTableReference.java diff --git a/src/sqlancer/doris/ast/DorisAggregateOperation.java b/src/sqlancer/doris/ast/DorisAggregateOperation.java index de818c26d..4c60985a8 100644 --- a/src/sqlancer/doris/ast/DorisAggregateOperation.java +++ b/src/sqlancer/doris/ast/DorisAggregateOperation.java @@ -3,14 +3,12 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.doris.visitor.DorisExprToNode; -public class DorisAggregateOperation extends - NewFunctionNode implements DorisExpression { +public class DorisAggregateOperation extends DorisFunction + implements DorisExpression { public DorisAggregateOperation(List args, DorisAggregateFunction func) { - super(DorisExprToNode.casts(args), func); + super(args, func); } public enum DorisAggregateFunction { diff --git a/src/sqlancer/doris/ast/DorisAlias.java b/src/sqlancer/doris/ast/DorisAlias.java new file mode 100644 index 000000000..74acf1cff --- /dev/null +++ b/src/sqlancer/doris/ast/DorisAlias.java @@ -0,0 +1,9 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class DorisAlias extends NewAliasNode implements DorisExpression { + public DorisAlias(DorisExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/doris/ast/DorisBetweenOperation.java b/src/sqlancer/doris/ast/DorisBetweenOperation.java index 76488393b..96c0b428c 100644 --- a/src/sqlancer/doris/ast/DorisBetweenOperation.java +++ b/src/sqlancer/doris/ast/DorisBetweenOperation.java @@ -2,23 +2,22 @@ import sqlancer.common.ast.newast.NewBetweenOperatorNode; import sqlancer.doris.DorisSchema; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisBetweenOperation extends NewBetweenOperatorNode implements DorisExpression { public DorisBetweenOperation(DorisExpression left, DorisExpression middle, DorisExpression right, boolean isTrue) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(middle), DorisExprToNode.cast(right), isTrue); + super(left, middle, right, isTrue); } public DorisExpression getLeftExpr() { - return (DorisExpression) left; + return left; } public DorisExpression getMiddleExpr() { - return (DorisExpression) middle; + return middle; } public DorisExpression getRightExpr() { - return (DorisExpression) right; + return right; } @Override diff --git a/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java index 244404569..2e148c488 100644 --- a/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java +++ b/src/sqlancer/doris/ast/DorisBinaryArithmeticOperation.java @@ -5,12 +5,11 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisBinaryArithmeticOperation extends NewBinaryOperatorNode implements DorisExpression { public DorisBinaryArithmeticOperation(DorisExpression left, DorisExpression right, BinaryOperatorNode.Operator op) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + super(left, right, op); } public enum DorisBinaryArithmeticOperator implements BinaryOperatorNode.Operator { @@ -114,11 +113,11 @@ public String getTextRepresentation() { } public DorisExpression getLeftExpr() { - return (DorisExpression) super.getLeft(); + return super.getLeft(); } public DorisExpression getRightExpr() { - return (DorisExpression) super.getRight(); + return super.getRight(); } public DorisBinaryArithmeticOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java index 894d2ab9b..13355c7ff 100644 --- a/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java +++ b/src/sqlancer/doris/ast/DorisBinaryComparisonOperation.java @@ -3,21 +3,20 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisBinaryComparisonOperation extends NewBinaryOperatorNode implements DorisExpression { public DorisBinaryComparisonOperation(DorisExpression left, DorisExpression right, DorisBinaryComparisonOperator op) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + super(left, right, op); } public DorisExpression getLeftExpression() { - return (DorisExpression) super.getLeft(); + return super.getLeft(); } public DorisExpression getRightExpression() { - return (DorisExpression) super.getRight(); + return super.getRight(); } public DorisBinaryComparisonOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java index b62ccd61b..858c3bf2f 100644 --- a/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java +++ b/src/sqlancer/doris/ast/DorisBinaryLogicalOperation.java @@ -4,20 +4,19 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisBinaryLogicalOperation extends NewBinaryOperatorNode implements DorisExpression { public DorisBinaryLogicalOperation(DorisExpression left, DorisExpression right, DorisBinaryLogicalOperator op) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + super(left, right, op); } public DorisExpression getLeftExpr() { - return (DorisExpression) super.getLeft(); + return super.getLeft(); } public DorisExpression getRightExpr() { - return (DorisExpression) super.getRight(); + return super.getRight(); } public DorisBinaryLogicalOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisBinaryOperation.java b/src/sqlancer/doris/ast/DorisBinaryOperation.java new file mode 100644 index 000000000..ffe80b86d --- /dev/null +++ b/src/sqlancer/doris/ast/DorisBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DorisBinaryOperation extends NewBinaryOperatorNode implements DorisExpression { + public DorisBinaryOperation(DorisExpression left, DorisExpression right, BinaryOperatorNode.Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/doris/ast/DorisCaseOperation.java b/src/sqlancer/doris/ast/DorisCaseOperation.java index e852e36bd..02641408a 100644 --- a/src/sqlancer/doris/ast/DorisCaseOperation.java +++ b/src/sqlancer/doris/ast/DorisCaseOperation.java @@ -2,9 +2,7 @@ import java.util.List; -import sqlancer.common.ast.newast.Node; - -public class DorisCaseOperation implements Node, DorisExpression { +public class DorisCaseOperation implements DorisExpression { private final DorisExpression expr; private final List conditions; diff --git a/src/sqlancer/doris/ast/DorisCastOperation.java b/src/sqlancer/doris/ast/DorisCastOperation.java index 1aacdaf06..e93b8b226 100644 --- a/src/sqlancer/doris/ast/DorisCastOperation.java +++ b/src/sqlancer/doris/ast/DorisCastOperation.java @@ -1,29 +1,28 @@ package sqlancer.doris.ast; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisSchema.DorisCompositeDataType; import sqlancer.doris.DorisSchema.DorisDataType; -public class DorisCastOperation implements Node, DorisExpression { - Node expr; +public class DorisCastOperation implements DorisExpression { + DorisExpression expr; DorisDataType type; - public DorisCastOperation(Node expr, DorisCompositeDataType type) { + public DorisCastOperation(DorisExpression expr, DorisCompositeDataType type) { this.expr = expr; this.type = type.getPrimitiveDataType(); } - public DorisCastOperation(Node expr, DorisDataType type) { + public DorisCastOperation(DorisExpression expr, DorisDataType type) { this.expr = expr; this.type = type; } - public Node getExpr() { + public DorisExpression getExpr() { return expr; } public DorisExpression getExpression() { - return (DorisExpression) expr; + return expr; } public DorisDataType getType() { diff --git a/src/sqlancer/doris/ast/DorisColumnReference.java b/src/sqlancer/doris/ast/DorisColumnReference.java new file mode 100644 index 000000000..ce6aee058 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.doris.DorisSchema; + +public class DorisColumnReference extends ColumnReferenceNode + implements DorisExpression { + public DorisColumnReference(DorisSchema.DorisColumn column) { + super(column); + } +} diff --git a/src/sqlancer/doris/ast/DorisConstant.java b/src/sqlancer/doris/ast/DorisConstant.java index 37374a4d7..2ad2500ed 100644 --- a/src/sqlancer/doris/ast/DorisConstant.java +++ b/src/sqlancer/doris/ast/DorisConstant.java @@ -1,10 +1,9 @@ package sqlancer.doris.ast; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.utils.DorisNumberUtils; -public abstract class DorisConstant implements Node, DorisExpression { +public abstract class DorisConstant implements DorisExpression { private DorisConstant() { } diff --git a/src/sqlancer/doris/ast/DorisFunction.java b/src/sqlancer/doris/ast/DorisFunction.java new file mode 100644 index 000000000..1f816a7e3 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisFunction.java @@ -0,0 +1,11 @@ +package sqlancer.doris.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DorisFunction extends NewFunctionNode implements DorisExpression { + public DorisFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/doris/ast/DorisFunctionOperation.java b/src/sqlancer/doris/ast/DorisFunctionOperation.java index 0a75892d0..05f5370b1 100644 --- a/src/sqlancer/doris/ast/DorisFunctionOperation.java +++ b/src/sqlancer/doris/ast/DorisFunctionOperation.java @@ -6,11 +6,10 @@ import java.util.stream.Stream; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.gen.DorisNewExpressionGenerator; -public class DorisFunctionOperation implements Node, DorisExpression { +public class DorisFunctionOperation implements DorisExpression { private DorisFunction function; private List args; diff --git a/src/sqlancer/doris/ast/DorisInOperation.java b/src/sqlancer/doris/ast/DorisInOperation.java index 31dcbe1ec..96ccd0998 100644 --- a/src/sqlancer/doris/ast/DorisInOperation.java +++ b/src/sqlancer/doris/ast/DorisInOperation.java @@ -4,7 +4,6 @@ import sqlancer.common.ast.newast.NewInOperatorNode; import sqlancer.doris.DorisSchema; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisInOperation extends NewInOperatorNode implements DorisExpression { @@ -12,7 +11,7 @@ public class DorisInOperation extends NewInOperatorNode impleme private final List rightExpr; public DorisInOperation(DorisExpression left, List right, boolean isNegated) { - super(DorisExprToNode.cast(left), DorisExprToNode.casts(right), isNegated); + super(left, right, isNegated); this.leftExpr = left; this.rightExpr = right; } diff --git a/src/sqlancer/doris/ast/DorisJoin.java b/src/sqlancer/doris/ast/DorisJoin.java index 01274449f..7c7a35c18 100644 --- a/src/sqlancer/doris/ast/DorisJoin.java +++ b/src/sqlancer/doris/ast/DorisJoin.java @@ -4,21 +4,17 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisColumn; -import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.gen.DorisNewExpressionGenerator; -import sqlancer.doris.visitor.DorisExprToNode; -public class DorisJoin implements Node { +public class DorisJoin implements DorisExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final DorisTableReference leftTable; + private final DorisTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final DorisExpression onCondition; public enum JoinType { INNER, STRAIGHT, LEFT, RIGHT; @@ -28,20 +24,19 @@ public static JoinType getRandom() { } } - public DorisJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public DorisJoin(DorisTableReference leftTable, DorisTableReference rightTable, JoinType joinType, + DorisExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public TableReferenceNode getLeftTable() { + public DorisTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public DorisTableReference getRightTable() { return rightTable; } @@ -49,35 +44,34 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public DorisExpression getOnCondition() { return onCondition; } - public static List> getJoins(List> tableList, - DorisGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DorisGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + DorisTableReference leftTable = tableList.remove(0); + DorisTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); DorisNewExpressionGenerator joinGen = new DorisNewExpressionGenerator(globalState).setColumns(columns); switch (DorisJoin.JoinType.getRandom()) { case INNER: joinExpressions.add(DorisJoin.createInnerJoin(leftTable, rightTable, - DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); break; case STRAIGHT: joinExpressions.add(DorisJoin.createStraightJoin(leftTable, rightTable, - DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); break; case LEFT: joinExpressions.add(DorisJoin.createLeftOuterJoin(leftTable, rightTable, - DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); break; case RIGHT: joinExpressions.add(DorisJoin.createRightOuterJoin(leftTable, rightTable, - DorisExprToNode.cast(joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)))); + joinGen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); break; default: throw new AssertionError(); @@ -86,23 +80,23 @@ public static List> getJoins(List left, - TableReferenceNode right, Node predicate) { + public static DorisJoin createInnerJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { return new DorisJoin(left, right, JoinType.INNER, predicate); } - public static DorisJoin createStraightJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DorisJoin createStraightJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { return new DorisJoin(left, right, JoinType.STRAIGHT, predicate); } - public static DorisJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DorisJoin createRightOuterJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { return new DorisJoin(left, right, JoinType.RIGHT, predicate); } - public static DorisJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DorisJoin createLeftOuterJoin(DorisTableReference left, DorisTableReference right, + DorisExpression predicate) { return new DorisJoin(left, right, JoinType.LEFT, predicate); } } diff --git a/src/sqlancer/doris/ast/DorisLikeOperation.java b/src/sqlancer/doris/ast/DorisLikeOperation.java index a3251ca7f..128a9299d 100644 --- a/src/sqlancer/doris/ast/DorisLikeOperation.java +++ b/src/sqlancer/doris/ast/DorisLikeOperation.java @@ -5,12 +5,11 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewBinaryOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisLikeOperation extends NewBinaryOperatorNode implements DorisExpression { public DorisLikeOperation(DorisExpression left, DorisExpression right, DorisLikeOperator op) { - super(DorisExprToNode.cast(left), DorisExprToNode.cast(right), op); + super(left, right, op); } @Override @@ -19,11 +18,11 @@ public DorisDataType getExpectedType() { } public DorisExpression getLeftExpr() { - return (DorisExpression) super.getLeft(); + return super.getLeft(); } public DorisExpression getRightExpr() { - return (DorisExpression) super.getRight(); + return super.getRight(); } public DorisLikeOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisOrderByTerm.java b/src/sqlancer/doris/ast/DorisOrderByTerm.java index 6207f4af2..01a7ab6d2 100644 --- a/src/sqlancer/doris/ast/DorisOrderByTerm.java +++ b/src/sqlancer/doris/ast/DorisOrderByTerm.java @@ -1,10 +1,9 @@ package sqlancer.doris.ast; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.Node; public class DorisOrderByTerm extends NewOrderingTerm implements DorisExpression { - public DorisOrderByTerm(Node expr, Ordering ordering) { + public DorisOrderByTerm(DorisExpression expr, Ordering ordering) { super(expr, ordering); } } diff --git a/src/sqlancer/doris/ast/DorisPostfixText.java b/src/sqlancer/doris/ast/DorisPostfixText.java new file mode 100644 index 000000000..889f4700b --- /dev/null +++ b/src/sqlancer/doris/ast/DorisPostfixText.java @@ -0,0 +1,9 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class DorisPostfixText extends NewPostfixTextNode implements DorisExpression { + public DorisPostfixText(DorisExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/doris/ast/DorisSelect.java b/src/sqlancer/doris/ast/DorisSelect.java index 283d474cc..6194a0498 100644 --- a/src/sqlancer/doris/ast/DorisSelect.java +++ b/src/sqlancer/doris/ast/DorisSelect.java @@ -2,9 +2,8 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class DorisSelect extends SelectBase> implements Node { +public class DorisSelect extends SelectBase implements DorisExpression { public enum DorisSelectDistinctType { diff --git a/src/sqlancer/doris/ast/DorisTableReference.java b/src/sqlancer/doris/ast/DorisTableReference.java new file mode 100644 index 000000000..b8a2cc2a2 --- /dev/null +++ b/src/sqlancer/doris/ast/DorisTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.doris.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.doris.DorisSchema; + +public class DorisTableReference extends TableReferenceNode + implements DorisExpression { + public DorisTableReference(DorisSchema.DorisTable table) { + super(table); + } +} diff --git a/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java index 9a34e2c54..f6961c7b7 100644 --- a/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java +++ b/src/sqlancer/doris/ast/DorisUnaryPostfixOperation.java @@ -4,17 +4,16 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements DorisExpression { public DorisUnaryPostfixOperation(DorisExpression expr, DorisUnaryPostfixOperator op) { - super(DorisExprToNode.cast(expr), op); + super(expr, op); } public DorisExpression getExpression() { - return (DorisExpression) getExpr(); + return getExpr(); } public DorisUnaryPostfixOperator getOp() { diff --git a/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java index e7f7d91b0..ffd919fdc 100644 --- a/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java +++ b/src/sqlancer/doris/ast/DorisUnaryPrefixOperation.java @@ -4,16 +4,15 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; import sqlancer.doris.DorisSchema.DorisDataType; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements DorisExpression { public DorisUnaryPrefixOperation(DorisExpression expr, DorisUnaryPrefixOperator op) { - super(DorisExprToNode.cast(expr), op); + super(expr, op); } public DorisExpression getExpression() { - return (DorisExpression) getExpr(); + return getExpr(); } public DorisUnaryPrefixOperator getOp() { diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java index 7ce4d2e60..50dc5cdec 100644 --- a/src/sqlancer/doris/gen/DorisInsertGenerator.java +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -11,7 +11,6 @@ import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisTable; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisInsertGenerator extends AbstractInsertGenerator { @@ -46,9 +45,8 @@ protected void insertValue(DorisColumn column) { if (column.hasDefaultValue() && Randomly.getBooleanWithRatherLowProbability()) { sb.append("DEFAULT"); } else { - String value = DorisToStringVisitor - .asString(DorisExprToNode.cast(new DorisNewExpressionGenerator(globalState) - .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()))); // 生成一个与column相同的常量类型 + String value = DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + .generateConstant(column.getType().getPrimitiveDataType(), column.isNullable())); // 生成一个与column相同的常量类型 sb.append(value); } } diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java index f283d0c39..13fe028ca 100644 --- a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -10,7 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.doris.DorisBugs; import sqlancer.doris.DorisProvider.DorisGlobalState; @@ -39,7 +38,6 @@ import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; -import sqlancer.doris.visitor.DorisExprToNode; public class DorisNewExpressionGenerator extends TypedExpressionGenerator { @@ -97,7 +95,7 @@ private DorisExpression createColumnOfType(DorisDataType type) { return DorisColumnValue.create(column, value); } - public List> generateOrderBy() { + public List generateOrderBy() { List randomColumns = Randomly.subset(columns); return randomColumns.stream() .map(c -> new DorisOrderByTerm(new DorisColumnValue(c, null), NewOrderingTerm.Ordering.getRandom())) @@ -121,7 +119,7 @@ public DorisExpression generateExpression(DorisDataType type, int depth) { } if (!DorisBugs.bug36070 && type != DorisDataType.NULL && globalState.getDbmsSpecificOptions().testCasts && Randomly.getBooleanWithRatherLowProbability()) { - return new DorisCastOperation(DorisExprToNode.cast(generateExpression(getRandomType(), depth + 1)), type); + return new DorisCastOperation(generateExpression(getRandomType(), depth + 1), type); } if (!DorisBugs.bug36070 && globalState.getDbmsSpecificOptions().testCase && Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java index 1b85d4b34..518a204a5 100644 --- a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java +++ b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java @@ -6,8 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisTable; @@ -17,7 +15,7 @@ import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisJoin; import sqlancer.doris.ast.DorisSelect; -import sqlancer.doris.visitor.DorisExprToNode; +import sqlancer.doris.ast.DorisTableReference; public final class DorisRandomQuerySynthesizer { @@ -33,33 +31,33 @@ public static DorisSelect generateSelect(DorisGlobalState globalState, int nrCol gen.setColumnOfLeafNode(columnOfLeafNode); int freeColumns = targetColumns.size(); select.setDistinct(DorisSelect.DorisSelectDistinctType.getRandomWithoutNull()); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { - Node column = null; + DorisExpression column = null; if (freeColumns > 0 && Randomly.getBoolean()) { column = new DorisColumnValue(targetColumns.get(freeColumns - 1), null); freeColumns -= 1; columnOfLeafNode.add((DorisColumnValue) column); } else { - column = DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); + column = gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN); } columns.add(column); } select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DorisJoin.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { - select.setHavingClause(DorisExprToNode.cast(gen.generateHavingClause())); + select.setHavingClause(gen.generateHavingClause()); } if (Randomly.getBoolean()) { - select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + select.setWhereClause(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); } - List> noExprColumns = new ArrayList<>(columnOfLeafNode); + List noExprColumns = new ArrayList<>(columnOfLeafNode); if (Randomly.getBoolean()) { select.setOrderByClauses(Randomly.nonEmptySubset(noExprColumns)); diff --git a/src/sqlancer/doris/gen/DorisTableGenerator.java b/src/sqlancer/doris/gen/DorisTableGenerator.java index 8a98739f5..74d2f956a 100644 --- a/src/sqlancer/doris/gen/DorisTableGenerator.java +++ b/src/sqlancer/doris/gen/DorisTableGenerator.java @@ -15,7 +15,6 @@ import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisCompositeDataType; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisTableGenerator { @@ -101,9 +100,8 @@ private static List getNewColumns(DorisGlobalState globalState) { && !isHllOrBitmap; String defaultValue = ""; if (hasDefaultValue) { - defaultValue = DorisToStringVisitor - .asString(DorisExprToNode.cast(new DorisNewExpressionGenerator(globalState) - .generateConstant(columnType.getPrimitiveDataType(), isNullable))); + defaultValue = DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + .generateConstant(columnType.getPrimitiveDataType(), isNullable)); } columns.add(new DorisColumn(columnName, columnType, iskey, isNullable, aggrType, hasDefaultValue, defaultValue)); diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java index 74fb9383d..906173921 100644 --- a/src/sqlancer/doris/gen/DorisUpdateGenerator.java +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -11,7 +11,6 @@ import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public final class DorisUpdateGenerator extends AbstractUpdateGenerator { @@ -45,7 +44,7 @@ private SQLQueryAdapter generate() { protected void updateValue(DorisColumn column) { if (Randomly.getBooleanWithSmallProbability()) { DorisExpression expr = gen.generateExpression(column.getType().getPrimitiveDataType()); - sb.append(DorisToStringVisitor.asString(DorisExprToNode.cast(expr))); + sb.append(DorisToStringVisitor.asString(expr)); } else { DorisExpression expr = gen.generateConstant(column.getType().getPrimitiveDataType(), column.isNullable()); sb.append(DorisToStringVisitor.asString(expr)); diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java index 01fa11f2f..8840a299e 100644 --- a/src/sqlancer/doris/oracle/DorisNoRECOracle.java +++ b/src/sqlancer/doris/oracle/DorisNoRECOracle.java @@ -11,10 +11,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -28,12 +24,14 @@ import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.DorisSchema.DorisTables; import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisPostfixText; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.gen.DorisNewExpressionGenerator; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisNoRECOracle extends NoRECBase implements TestOracle { @@ -52,12 +50,11 @@ public void check() throws SQLException { DorisTables randomTables = s.getRandomTableNonEmptyTables(); List columns = randomTables.getColumns(); DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = DorisExprToNode - .cast(gen.generateExpression(DorisDataType.BOOLEAN)); + DorisExpression randomWhereCondition = gen.generateExpression(DorisDataType.BOOLEAN); List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DorisJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, state); int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, @@ -71,11 +68,11 @@ public void check() throws SQLException { } } - private int getUnoptimizedQueryCount(List> tableList, - Node randomWhereCondition, List> joins) throws SQLException { + private int getUnoptimizedQueryCount(List tableList, DorisExpression randomWhereCondition, + List joins) throws SQLException { DorisSelect select = new DorisSelect(); - Node asText = new NewPostfixTextNode<>(new DorisCastOperation( - new NewPostfixTextNode(randomWhereCondition, + DorisExpression asText = new DorisPostfixText(new DorisCastOperation( + new DorisPostfixText(randomWhereCondition, " IS NOT NULL AND " + DorisToStringVisitor.asString(randomWhereCondition)), new DorisCompositeDataType(DorisDataType.INT, 8)), "as count"); select.setFetchColumns(Arrays.asList(asText)); @@ -100,18 +97,17 @@ private int getUnoptimizedQueryCount(List> tableList, return secondCount; } - private int getOptimizedQueryCount(SQLConnection con, List> tableList, - List columns, Node randomWhereCondition, List> joins) - throws SQLException { + private int getOptimizedQueryCount(SQLConnection con, List tableList, List columns, + DorisExpression randomWhereCondition, List joins) throws SQLException { DorisSelect select = new DorisSelect(); // select.setGroupByClause(groupBys); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List allColumns = columns.stream().map((c) -> new DorisColumnReference(c)) + .collect(Collectors.toList()); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); if (Randomly.getBooleanWithSmallProbability()) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); diff --git a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java index 4d8f3c012..a7e8b5b02 100644 --- a/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/doris/oracle/DorisPivotedQuerySynthesisOracle.java @@ -7,8 +7,6 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.PivotedQuerySynthesisBase; import sqlancer.common.query.Query; import sqlancer.common.query.SQLQueryAdapter; @@ -17,17 +15,16 @@ import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.DorisSchema.DorisRowValue; -import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.DorisSchema.DorisTables; import sqlancer.doris.ast.DorisColumnValue; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.ast.DorisUnaryPostfixOperation; import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.gen.DorisNewExpressionGenerator; import sqlancer.doris.visitor.DorisExpectedValueVisitor; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisPivotedQuerySynthesisOracle @@ -53,22 +50,22 @@ protected Query getRectifiedQuery() throws Exception { selectStatement.setFetchColumns(fetchColumns.stream() .map(c -> new DorisColumnValue(getFetchValueAliasedColumn(c), pivotRow.getValues().get(c))) .collect(Collectors.toList())); - selectStatement.setFromList(randomTables.getTables().stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList())); + selectStatement.setFromList( + randomTables.getTables().stream().map(t -> new DorisTableReference(t)).collect(Collectors.toList())); DorisExpression whereClause = generateRectifiedExpression(columns, pivotRow); - selectStatement.setWhereClause(DorisExprToNode.cast(whereClause)); - List> groupByClause = generateGroupByClause(columns, pivotRow); + selectStatement.setWhereClause(whereClause); + List groupByClause = generateGroupByClause(columns, pivotRow); selectStatement.setGroupByExpressions(groupByClause); - Node limitClause = generateLimit(); + DorisExpression limitClause = generateLimit(); selectStatement.setLimitClause(limitClause); if (limitClause != null) { - Node offsetClause = generateOffset(); + DorisExpression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); gen.setColumns(columns); if (!isDistinct) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add(new DorisConstant.DorisIntConstant( Randomly.smallNumber() % selectStatement.getFetchColumns().size() + 1)); selectStatement.setOrderByClauses(constants); @@ -124,10 +121,10 @@ private DorisColumn getFetchValueAliasedColumn(DorisColumn c) { @Override protected String getExpectedValues(DorisExpression expr) { - return DorisExpectedValueVisitor.asExpectedValues(DorisExprToNode.cast(expr)); + return DorisExpectedValueVisitor.asExpectedValues(expr); } - private List> generateGroupByClause(List columns, DorisRowValue rowValue) { + private List generateGroupByClause(List columns, DorisRowValue rowValue) { if (Randomly.getBoolean()) { return columns.stream().map(c -> new DorisColumnValue(c, rowValue.getValues().get(c))) .collect(Collectors.toList()); @@ -136,7 +133,7 @@ private List> generateGroupByClause(List colu } } - private Node generateLimit() { + private DorisExpression generateLimit() { if (Randomly.getBoolean()) { return DorisConstant.createIntConstant(Integer.MAX_VALUE); } else { @@ -144,7 +141,7 @@ private Node generateLimit() { } } - private Node generateOffset() { + private DorisExpression generateOffset() { if (Randomly.getBoolean()) { return DorisConstant.createIntConstant(0); } else { diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java index 1f3fe66f1..b3289e5ad 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -8,12 +8,6 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; @@ -22,15 +16,19 @@ import sqlancer.doris.DorisSchema.DorisCompositeDataType; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.ast.DorisAggregateOperation; +import sqlancer.doris.ast.DorisAlias; import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; import sqlancer.doris.ast.DorisBinaryArithmeticOperation; +import sqlancer.doris.ast.DorisBinaryOperation; import sqlancer.doris.ast.DorisCastOperation; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunction; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningAggregateTester extends DorisQueryPartitioningBase @@ -53,16 +51,16 @@ public void check() throws SQLException { DorisAggregateFunction aggregateFunction = Randomly.fromOptions(DorisAggregateFunction.MAX, DorisAggregateFunction.MIN, DorisAggregateFunction.SUM, DorisAggregateFunction.COUNT, DorisAggregateFunction.AVG); - NewFunctionNode aggregate = (DorisAggregateOperation) gen + DorisFunction aggregate = (DorisAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); - List> fetchColumns = new ArrayList<>(); + List fetchColumns = new ArrayList<>(); fetchColumns.add(aggregate); while (Randomly.getBooleanWithRatherLowProbability()) { fetchColumns.add((DorisAggregateOperation) gen.generateAggregate()); } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); @@ -94,15 +92,13 @@ public void check() throws SQLException { } - private String createMetamorphicUnionQuery(DorisSelect select, - NewFunctionNode aggregate, List> from) { + private String createMetamorphicUnionQuery(DorisSelect select, DorisFunction aggregate, + List from) { String metamorphicQuery; - Node whereClause = DorisExprToNode.cast(gen.generateExpression(DorisDataType.BOOLEAN)); - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, - DorisUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, - DorisUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); + DorisExpression whereClause = gen.generateExpression(DorisDataType.BOOLEAN); + DorisExpression negatedClause = new DorisUnaryPrefixOperation(whereClause, DorisUnaryPrefixOperator.NOT); + DorisExpression notNullClause = new DorisUnaryPostfixOperation(whereClause, DorisUnaryPostfixOperator.IS_NULL); + List mappedAggregate = mapped(aggregate); DorisSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); DorisSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); DorisSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); @@ -141,7 +137,7 @@ private String getAggregateResult(String queryString) throws SQLException { } } - private List> mapped(NewFunctionNode aggregate) { + private List mapped(DorisFunction aggregate) { DorisCastOperation count; switch (aggregate.getFunc()) { @@ -151,21 +147,19 @@ private List> mapped(NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + DorisFunction sum = new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.SUM); - count = new DorisCastOperation(new NewFunctionNode<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), + count = new DorisCastOperation(new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), new DorisCompositeDataType(DorisDataType.FLOAT, 8)); return aliasArgs(Arrays.asList(sum, count)); case STDDEV_POP: - NewFunctionNode sumSquared = new NewFunctionNode<>( - Arrays.asList(new NewBinaryOperatorNode<>(aggregate.getArgs().get(0), aggregate.getArgs().get(0), + DorisFunction sumSquared = new DorisFunction<>( + Arrays.asList(new DorisBinaryOperation(aggregate.getArgs().get(0), aggregate.getArgs().get(0), DorisBinaryArithmeticOperation.DorisBinaryArithmeticOperator.MULTIPLICATION)), DorisAggregateFunction.SUM); - count = new DorisCastOperation( - new NewFunctionNode(aggregate.getArgs(), - DorisAggregateFunction.COUNT), + count = new DorisCastOperation(new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.COUNT), new DorisCompositeDataType(DorisDataType.FLOAT, 8)); - NewFunctionNode avg = new NewFunctionNode<>(aggregate.getArgs(), + DorisFunction avg = new DorisFunction<>(aggregate.getArgs(), DorisAggregateFunction.AVG); return aliasArgs(Arrays.asList(sumSquared, count, avg)); default: @@ -173,16 +167,16 @@ private List> mapped(NewFunctionNode> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode(expr, "agg" + i++)); + for (DorisExpression expr : originalAggregateArgs) { + args.add(new DorisAlias(expr, "agg" + i++)); } return args; } - private String getOuterAggregateFunction(NewFunctionNode aggregate) { + private String getOuterAggregateFunction(DorisFunction aggregate) { switch (aggregate.getFunc()) { case STDDEV_POP: return "sqrt(SUM(agg0)/SUM(agg1)-SUM(agg2)*SUM(agg2))"; @@ -195,8 +189,8 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, - Node whereClause, List> joinList) { + private DorisSelect getSelect(List aggregates, List from, + DorisExpression whereClause, List joinList) { DorisSelect leftSelect = new DorisSelect(); leftSelect.setFetchColumns(aggregates); leftSelect.setFromList(from); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java index defc76a56..b3d3a7a6b 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java @@ -7,9 +7,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -19,10 +16,12 @@ import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.DorisSchema.DorisTables; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisColumnValue; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisJoin; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.gen.DorisNewExpressionGenerator; public class DorisQueryPartitioningBase extends TernaryLogicPartitioningOracleBase @@ -33,7 +32,7 @@ public class DorisQueryPartitioningBase extends TernaryLogicPartitioningOracleBa DorisNewExpressionGenerator gen; DorisSelect select; - List> groupByExpression; + List groupByExpression; public DorisQueryPartitioningBase(DorisGlobalState state) { super(state); @@ -56,21 +55,21 @@ public void check() throws SQLException { groupByExpression = new ArrayList<>(allColumnValues); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = DorisJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new DorisColumn("*", null, false, false))); + columns.add(new DorisColumnReference(new DorisColumn("*", null, false, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DorisColumnReference(c)) + .collect(Collectors.toList()); } return columns; } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java index 3bce110ff..dc350ea6f 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningDistinctTester.java @@ -7,7 +7,6 @@ import sqlancer.ComparatorHelper; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningDistinctTester extends DorisQueryPartitioningBase { @@ -26,11 +25,11 @@ public void check() throws SQLException { String originalQueryString = DorisToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setWhereClause(DorisExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DorisToStringVisitor.asString(select); List combinedString = new ArrayList<>(); diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java index 8e173d7ae..97ade18e3 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningGroupByTester.java @@ -7,13 +7,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningGroupByTester extends DorisQueryPartitioningBase { @@ -33,11 +30,11 @@ public void check() throws SQLException { List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setWhereClause(DorisExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DorisToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSetNoDuplicates(firstQueryString, @@ -47,9 +44,9 @@ public void check() throws SQLException { } @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new DorisColumnReference(c)) + .collect(Collectors.toList()); } } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java index b3398f229..434ee01d5 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningHavingTester.java @@ -6,14 +6,12 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningHavingTester extends DorisQueryPartitioningBase @@ -29,12 +27,12 @@ public DorisQueryPartitioningHavingTester(DorisGlobalState state) { public void check() throws SQLException { super.check(); if (Randomly.getBoolean()) { - select.setWhereClause(DorisExprToNode.cast(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + select.setWhereClause(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN)); } select.setFetchColumns(groupByExpression); boolean orderBy = Randomly.getBoolean(); if (orderBy) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); @@ -44,11 +42,11 @@ public void check() throws SQLException { String originalQueryString = DorisToStringVisitor.asString(select); List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - select.setHavingClause(DorisExprToNode.cast(predicate)); + select.setHavingClause(predicate); String firstQueryString = DorisToStringVisitor.asString(select); - select.setHavingClause(DorisExprToNode.cast(negatedPredicate)); + select.setHavingClause(negatedPredicate); String secondQueryString = DorisToStringVisitor.asString(select); - select.setHavingClause(DorisExprToNode.cast(isNullPredicate)); + select.setHavingClause(isNullPredicate); String thirdQueryString = DorisToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, @@ -63,10 +61,9 @@ protected DorisExpression generatePredicate() { } @Override - List> generateFetchColumns() { + List generateFetchColumns() { gen.setAllowAggregateFunctions(true); - List> expressions = DorisExprToNode - .casts(gen.generateExpressions(Randomly.smallNumber() + 1)); + List expressions = gen.generateExpressions(Randomly.smallNumber() + 1); gen.setAllowAggregateFunctions(false); return expressions; } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java index 34ade1d3a..5a593a573 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -6,12 +6,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisExprToNode; import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisQueryPartitioningWhereTester extends DorisQueryPartitioningBase { @@ -32,16 +30,16 @@ public void check() throws SQLException { boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); if (orderBy) { - List> constants = new ArrayList<>(); + List constants = new ArrayList<>(); constants.add( new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); select.setOrderByClauses(constants); } - select.setWhereClause(DorisExprToNode.cast(predicate)); + select.setWhereClause(predicate); String firstQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(negatedPredicate)); + select.setWhereClause(negatedPredicate); String secondQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(DorisExprToNode.cast(isNullPredicate)); + select.setWhereClause(isNullPredicate); String thirdQueryString = DorisToStringVisitor.asString(select); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, diff --git a/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java index 57cb29ae0..10c90e47e 100644 --- a/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java +++ b/src/sqlancer/doris/visitor/DorisExpectedValueVisitor.java @@ -2,60 +2,57 @@ import java.util.List; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.ast.DorisAlias; +import sqlancer.doris.ast.DorisBetweenOperation; +import sqlancer.doris.ast.DorisBinaryOperation; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; +import sqlancer.doris.ast.DorisFunction; +import sqlancer.doris.ast.DorisInOperation; import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisOrderByTerm; +import sqlancer.doris.ast.DorisPostfixText; import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; +import sqlancer.doris.ast.DorisUnaryPostfixOperation; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; public class DorisExpectedValueVisitor { protected final StringBuilder sb = new StringBuilder(); - private void print(Node expr) { + private void print(DorisExpression expr) { sb.append(DorisToStringVisitor.asString(expr)); sb.append(" -- "); sb.append(((DorisExpression) expr).getExpectedValue()); sb.append("\n"); } - @SuppressWarnings("unchecked") - public void visit(Node expr) { + public void visit(DorisExpression expr) { assert expr != null; - if (expr instanceof ColumnReferenceNode) { - visit((ColumnReferenceNode) expr); - } else if (expr instanceof NewUnaryPostfixOperatorNode) { - visit((NewUnaryPostfixOperatorNode) expr); - } else if (expr instanceof NewUnaryPrefixOperatorNode) { - visit((NewUnaryPrefixOperatorNode) expr); - } else if (expr instanceof NewBinaryOperatorNode) { - visit((NewBinaryOperatorNode) expr); - } else if (expr instanceof TableReferenceNode) { - visit((TableReferenceNode) expr); - } else if (expr instanceof NewFunctionNode) { - visit((NewFunctionNode) expr); - } else if (expr instanceof NewBetweenOperatorNode) { - visit((NewBetweenOperatorNode) expr); - } else if (expr instanceof NewInOperatorNode) { - visit((NewInOperatorNode) expr); - } else if (expr instanceof NewOrderingTerm) { - visit((NewOrderingTerm) expr); - } else if (expr instanceof NewAliasNode) { - visit((NewAliasNode) expr); - } else if (expr instanceof NewPostfixTextNode) { - visit((NewPostfixTextNode) expr); + if (expr instanceof DorisColumnReference) { + visit((DorisColumnReference) expr); + } else if (expr instanceof DorisUnaryPostfixOperation) { + visit((DorisUnaryPostfixOperation) expr); + } else if (expr instanceof DorisUnaryPrefixOperation) { + visit((DorisUnaryPrefixOperation) expr); + } else if (expr instanceof DorisBinaryOperation) { + visit((DorisBinaryOperation) expr); + } else if (expr instanceof DorisTableReference) { + visit((DorisTableReference) expr); + } else if (expr instanceof DorisFunction) { + visit((DorisFunction) expr); + } else if (expr instanceof DorisBetweenOperation) { + visit((DorisBetweenOperation) expr); + } else if (expr instanceof DorisInOperation) { + visit((DorisInOperation) expr); + } else if (expr instanceof DorisOrderByTerm) { + visit((DorisOrderByTerm) expr); + } else if (expr instanceof DorisAlias) { + visit((DorisAlias) expr); + } else if (expr instanceof DorisPostfixText) { + visit((DorisPostfixText) expr); } else if (expr instanceof DorisConstant) { visit((DorisConstant) expr); } else if (expr instanceof DorisSelect) { @@ -67,65 +64,64 @@ public void visit(Node expr) { } } - public void visit(ColumnReferenceNode c) { + public void visit(DorisColumnReference c) { print(c); } - public void visit(NewUnaryPostfixOperatorNode op) { + public void visit(DorisUnaryPostfixOperation op) { print(op); visit(op.getExpr()); } - public void visit(NewUnaryPrefixOperatorNode op) { + public void visit(DorisUnaryPrefixOperation op) { print(op); visit(op.getExpr()); } - public void visit(NewBinaryOperatorNode op) { - print(op); + public void visit(DorisBinaryOperation op) { visit(op.getLeft()); visit(op.getRight()); } - public void visit(TableReferenceNode t) { + public void visit(DorisTableReference t) { print(t); } - public void visit(NewFunctionNode fun) { + public void visit(DorisFunction fun) { print(fun); visit(fun.getArgs()); } - public void visit(List> expressions) { - for (Node expression : expressions) { + public void visit(List expressions) { + for (DorisExpression expression : expressions) { visit(expression); } } - public void visit(NewBetweenOperatorNode op) { + public void visit(DorisBetweenOperation op) { print(op); visit(op.getLeft()); visit(op.getMiddle()); visit(op.getRight()); } - public void visit(NewInOperatorNode op) { + public void visit(DorisInOperation op) { print(op); visit(op.getLeft()); visit(op.getRight()); } - public void visit(NewOrderingTerm op) { + public void visit(DorisOrderByTerm op) { print(op); visit(op.getExpr()); } - public void visit(NewAliasNode op) { + public void visit(DorisAlias op) { print(op); visit(op.getExpr()); } - public void visit(NewPostfixTextNode postFixText) { + public void visit(DorisPostfixText postFixText) { print(postFixText); visit(postFixText.getExpr()); } @@ -146,7 +142,7 @@ public String get() { return sb.toString(); } - public static String asExpectedValues(Node expr) { + public static String asExpectedValues(DorisExpression expr) { DorisExpectedValueVisitor v = new DorisExpectedValueVisitor(); v.visit(expr); return v.get(); diff --git a/src/sqlancer/doris/visitor/DorisToStringVisitor.java b/src/sqlancer/doris/visitor/DorisToStringVisitor.java index f7770058f..fcea9ab69 100644 --- a/src/sqlancer/doris/visitor/DorisToStringVisitor.java +++ b/src/sqlancer/doris/visitor/DorisToStringVisitor.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.doris.ast.DorisCaseOperation; import sqlancer.doris.ast.DorisCastOperation; import sqlancer.doris.ast.DorisConstant; @@ -14,7 +13,7 @@ public class DorisToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(DorisExpression expr) { if (expr instanceof DorisConstant) { visit((DorisConstant) expr); } else if (expr instanceof DorisSelect) { @@ -34,7 +33,7 @@ public void visitSpecific(Node expr) { private void visit(DorisJoin join) { sb.append(" "); - visit(join.getLeftTable()); + visit((DorisExpression) join.getLeftTable()); sb.append(" "); switch (join.getJoinType()) { case INNER: @@ -65,7 +64,7 @@ private void visit(DorisJoin join) { default: throw new AssertionError(); } - visit(join.getRightTable()); + visit((DorisExpression) join.getRightTable()); sb.append(" "); if (join.getOnCondition() != null) { sb.append("ON "); @@ -91,7 +90,7 @@ private void visit(DorisFunctionOperation func) { if (func.getArgs() != null) { for (int i = 0; i < func.getArgs().size(); i++) { - visit(DorisExprToNode.cast(func.getArgs().get(i))); + visit(func.getArgs().get(i)); if (i != func.getArgs().size() - 1) { sb.append(","); } @@ -102,20 +101,20 @@ private void visit(DorisFunctionOperation func) { private void visit(DorisCaseOperation cases) { sb.append("CASE "); - visit(DorisExprToNode.cast(cases.getExpr())); + visit(cases.getExpr()); sb.append(" "); for (int i = 0; i < cases.getConditions().size(); i++) { DorisExpression predicate = cases.getConditions().get(i); DorisExpression then = cases.getThenClauses().get(i); sb.append(" WHEN "); - visit(DorisExprToNode.cast(predicate)); + visit(predicate); sb.append(" THEN "); - visit(DorisExprToNode.cast(then)); + visit(then); sb.append(" "); } if (cases.getElseClause() != null) { sb.append("ELSE "); - visit(DorisExprToNode.cast(cases.getElseClause())); + visit(cases.getElseClause()); sb.append(" "); } sb.append("END "); @@ -161,15 +160,9 @@ private void visit(DorisSelect select) { } } - public static String asString(Node expr) { - DorisToStringVisitor visitor = new DorisToStringVisitor(); - visitor.visit(expr); - return visitor.get(); - } - public static String asString(DorisExpression expr) { DorisToStringVisitor visitor = new DorisToStringVisitor(); - visitor.visit(DorisExprToNode.cast(expr)); + visitor.visit(expr); return visitor.get(); } } From bc08478432b34c25bd0689c3df0c93cd08770227 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 10:03:42 +0800 Subject: [PATCH 473/856] [H2] Update expressions to match new AST --- src/sqlancer/h2/H2CastNode.java | 24 ------- src/sqlancer/h2/H2ExpressionGenerator.java | 65 +++++++++---------- src/sqlancer/h2/H2QueryPartitioningBase.java | 24 +++---- src/sqlancer/h2/H2RandomQuerySynthesizer.java | 17 +++-- src/sqlancer/h2/H2Select.java | 8 --- src/sqlancer/h2/H2ToStringVisitor.java | 14 ++-- src/sqlancer/h2/ast/H2BetweenOperation.java | 9 +++ src/sqlancer/h2/ast/H2BinaryOperation.java | 10 +++ src/sqlancer/h2/ast/H2CaseOperation.java | 12 ++++ src/sqlancer/h2/ast/H2CastNode.java | 23 +++++++ src/sqlancer/h2/ast/H2ColumnReference.java | 11 ++++ src/sqlancer/h2/{ => ast}/H2Constant.java | 18 +++-- src/sqlancer/h2/{ => ast}/H2Expression.java | 2 +- src/sqlancer/h2/ast/H2Function.java | 11 ++++ src/sqlancer/h2/ast/H2InOperation.java | 11 ++++ src/sqlancer/h2/{ => ast}/H2Join.java | 46 ++++++------- src/sqlancer/h2/ast/H2Select.java | 7 ++ src/sqlancer/h2/ast/H2TableReference.java | 10 +++ .../h2/ast/H2UnaryPostfixOperation.java | 10 +++ .../h2/ast/H2UnaryPrefixOperation.java | 10 +++ 20 files changed, 215 insertions(+), 127 deletions(-) delete mode 100644 src/sqlancer/h2/H2CastNode.java delete mode 100644 src/sqlancer/h2/H2Select.java create mode 100644 src/sqlancer/h2/ast/H2BetweenOperation.java create mode 100644 src/sqlancer/h2/ast/H2BinaryOperation.java create mode 100644 src/sqlancer/h2/ast/H2CaseOperation.java create mode 100644 src/sqlancer/h2/ast/H2CastNode.java create mode 100644 src/sqlancer/h2/ast/H2ColumnReference.java rename src/sqlancer/h2/{ => ast}/H2Constant.java (83%) rename src/sqlancer/h2/{ => ast}/H2Expression.java (59%) create mode 100644 src/sqlancer/h2/ast/H2Function.java create mode 100644 src/sqlancer/h2/ast/H2InOperation.java rename src/sqlancer/h2/{ => ast}/H2Join.java (55%) create mode 100644 src/sqlancer/h2/ast/H2Select.java create mode 100644 src/sqlancer/h2/ast/H2TableReference.java create mode 100644 src/sqlancer/h2/ast/H2UnaryPostfixOperation.java create mode 100644 src/sqlancer/h2/ast/H2UnaryPrefixOperation.java diff --git a/src/sqlancer/h2/H2CastNode.java b/src/sqlancer/h2/H2CastNode.java deleted file mode 100644 index 9d279c8dd..000000000 --- a/src/sqlancer/h2/H2CastNode.java +++ /dev/null @@ -1,24 +0,0 @@ -package sqlancer.h2; - -import sqlancer.common.ast.newast.Node; -import sqlancer.h2.H2Schema.H2CompositeDataType; - -public class H2CastNode implements Node { - - private final Node expression; - private final H2CompositeDataType type; - - public H2CastNode(Node expression, H2CompositeDataType type) { - this.expression = expression; - this.type = type; - } - - public Node getExpression() { - return expression; - } - - public H2CompositeDataType getType() { - return type; - } - -} diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index d04251409..d4a2240e6 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -2,22 +2,23 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewCaseOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2CompositeDataType; import sqlancer.h2.H2Schema.H2DataType; - -public class H2ExpressionGenerator extends UntypedExpressionGenerator, H2Column> { +import sqlancer.h2.ast.H2BetweenOperation; +import sqlancer.h2.ast.H2BinaryOperation; +import sqlancer.h2.ast.H2CaseOperation; +import sqlancer.h2.ast.H2CastNode; +import sqlancer.h2.ast.H2ColumnReference; +import sqlancer.h2.ast.H2Constant; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2InOperation; +import sqlancer.h2.ast.H2UnaryPostfixOperation; +import sqlancer.h2.ast.H2UnaryPrefixOperation; + +public class H2ExpressionGenerator extends UntypedExpressionGenerator { private final H2GlobalState globalState; @@ -31,7 +32,7 @@ private enum Expression { } @Override - protected Node generateExpression(int depth) { + protected H2Expression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); } @@ -39,37 +40,33 @@ protected Node generateExpression(int depth) { switch (expr) { case BINARY_COMPARISON: Operator op = H2BinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), - op); + return new H2BinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_LOGICAL: op = H2BinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), - op); + return new H2BinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); case UNARY_POSTFIX: op = H2UnaryPostfixOperator.getRandom(); - return new NewUnaryPostfixOperatorNode(generateExpression(depth + 1), op); + return new H2UnaryPostfixOperation(generateExpression(depth + 1), op); case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode(generateExpression(depth + 1), - H2UnaryPrefixOperator.getRandom()); + return new H2UnaryPrefixOperation(generateExpression(depth + 1), H2UnaryPrefixOperator.getRandom()); case IN: - return new NewInOperatorNode(generateExpression(depth + 1), + return new H2InOperation(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); case BETWEEN: - return new NewBetweenOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + return new H2BetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); case CASE: int nr = Randomly.smallNumber() + 1; - return new NewCaseOperatorNode(generateExpression(depth + 1), - generateExpressions(nr, depth + 1), generateExpressions(nr, depth + 1), - generateExpression(depth + 1)); + return new H2CaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode(generateExpression(depth + 1), generateExpression(depth + 1), + return new H2BinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), H2BinaryArithmeticOperator.getRandom()); case CAST: return new H2CastNode(generateExpression(depth + 1), H2CompositeDataType.getRandom()); case FUNCTION: H2Function func = H2Function.getRandom(); - return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + return new sqlancer.h2.ast.H2Function<>(generateExpressions(func.getNrArgs()), func); default: throw new AssertionError(); } @@ -203,12 +200,12 @@ public int getNrArgs() { } @Override - protected Node generateColumn() { - return new ColumnReferenceNode(Randomly.fromList(columns)); + protected H2Expression generateColumn() { + return new H2ColumnReference(Randomly.fromList(columns)); } @Override - public Node generateConstant() { + public H2Expression generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return H2Constant.createNullConstant(); } @@ -330,13 +327,13 @@ public String getTextRepresentation() { } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, H2UnaryPrefixOperator.NOT); + public H2Expression negatePredicate(H2Expression predicate) { + return new H2UnaryPrefixOperation(predicate, H2UnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, H2UnaryPostfixOperator.IS_NULL); + public H2Expression isNull(H2Expression expr) { + return new H2UnaryPostfixOperation(expr, H2UnaryPostfixOperator.IS_NULL); } } diff --git a/src/sqlancer/h2/H2QueryPartitioningBase.java b/src/sqlancer/h2/H2QueryPartitioningBase.java index ec5af4e2e..34351f58f 100644 --- a/src/sqlancer/h2/H2QueryPartitioningBase.java +++ b/src/sqlancer/h2/H2QueryPartitioningBase.java @@ -5,9 +5,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -15,8 +12,13 @@ import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2Table; import sqlancer.h2.H2Schema.H2Tables; +import sqlancer.h2.ast.H2ColumnReference; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; +import sqlancer.h2.ast.H2TableReference; -public class H2QueryPartitioningBase extends TernaryLogicPartitioningOracleBase, H2GlobalState> +public class H2QueryPartitioningBase extends TernaryLogicPartitioningOracleBase implements TestOracle { H2Schema s; @@ -38,22 +40,22 @@ public void check() throws SQLException { select = new H2Select(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = H2Join.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new H2TableReference(t)) + .collect(Collectors.toList()); + List joins = H2Join.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); - columns.add(new ColumnReferenceNode<>(new H2Column("*", null))); + List generateFetchColumns() { + List columns = new ArrayList<>(); + columns.add(new H2ColumnReference(new H2Column("*", null))); return columns; } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } diff --git a/src/sqlancer/h2/H2RandomQuerySynthesizer.java b/src/sqlancer/h2/H2RandomQuerySynthesizer.java index dafa7835c..ba89bfd0b 100644 --- a/src/sqlancer/h2/H2RandomQuerySynthesizer.java +++ b/src/sqlancer/h2/H2RandomQuerySynthesizer.java @@ -5,11 +5,14 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; import sqlancer.h2.H2Schema.H2Tables; +import sqlancer.h2.ast.H2Constant; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; +import sqlancer.h2.ast.H2TableReference; public final class H2RandomQuerySynthesizer { @@ -20,16 +23,16 @@ public static H2Select generateSelect(H2GlobalState globalState, int nrColumns) H2Tables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState).setColumns(targetTables.getColumns()); H2Select select = new H2Select(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { - Node expression = gen.generateExpression(); + H2Expression expression = gen.generateExpression(); columns.add(expression); } select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = H2Join.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new H2TableReference(t)) + .collect(Collectors.toList()); + List joins = H2Join.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/h2/H2Select.java b/src/sqlancer/h2/H2Select.java deleted file mode 100644 index 07191659d..000000000 --- a/src/sqlancer/h2/H2Select.java +++ /dev/null @@ -1,8 +0,0 @@ -package sqlancer.h2; - -import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; - -public class H2Select extends SelectBase> implements Node { - -} diff --git a/src/sqlancer/h2/H2ToStringVisitor.java b/src/sqlancer/h2/H2ToStringVisitor.java index ad45c66f2..ab18ab235 100644 --- a/src/sqlancer/h2/H2ToStringVisitor.java +++ b/src/sqlancer/h2/H2ToStringVisitor.java @@ -1,12 +1,16 @@ package sqlancer.h2; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; +import sqlancer.h2.ast.H2CastNode; +import sqlancer.h2.ast.H2Constant; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; public class H2ToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(H2Expression expr) { if (expr instanceof H2Constant) { visit((H2Constant) expr); } else if (expr instanceof H2Select) { @@ -33,11 +37,11 @@ private void visit(H2CastNode cast) { } private void visit(H2Join join) { - visit(join.getLeftTable()); + visit((H2Expression) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" JOIN "); - visit(join.getRightTable()); + visit((H2Expression) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); @@ -81,7 +85,7 @@ public void visit(H2Select select) { } } - public static String asString(Node expr) { + public static String asString(H2Expression expr) { H2ToStringVisitor visitor = new H2ToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/h2/ast/H2BetweenOperation.java b/src/sqlancer/h2/ast/H2BetweenOperation.java new file mode 100644 index 000000000..77921d91e --- /dev/null +++ b/src/sqlancer/h2/ast/H2BetweenOperation.java @@ -0,0 +1,9 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class H2BetweenOperation extends NewBetweenOperatorNode implements H2Expression { + public H2BetweenOperation(H2Expression left, H2Expression middle, H2Expression right, boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/h2/ast/H2BinaryOperation.java b/src/sqlancer/h2/ast/H2BinaryOperation.java new file mode 100644 index 000000000..d523256fd --- /dev/null +++ b/src/sqlancer/h2/ast/H2BinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class H2BinaryOperation extends NewBinaryOperatorNode implements H2Expression { + public H2BinaryOperation(H2Expression left, H2Expression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/h2/ast/H2CaseOperation.java b/src/sqlancer/h2/ast/H2CaseOperation.java new file mode 100644 index 000000000..75e51fc4d --- /dev/null +++ b/src/sqlancer/h2/ast/H2CaseOperation.java @@ -0,0 +1,12 @@ +package sqlancer.h2.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class H2CaseOperation extends NewCaseOperatorNode implements H2Expression { + public H2CaseOperation(H2Expression switchCondition, List conditions, List expressions, + H2Expression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/h2/ast/H2CastNode.java b/src/sqlancer/h2/ast/H2CastNode.java new file mode 100644 index 000000000..5e7602311 --- /dev/null +++ b/src/sqlancer/h2/ast/H2CastNode.java @@ -0,0 +1,23 @@ +package sqlancer.h2.ast; + +import sqlancer.h2.H2Schema.H2CompositeDataType; + +public class H2CastNode implements H2Expression { + + private final H2Expression expression; + private final H2CompositeDataType type; + + public H2CastNode(H2Expression expression, H2CompositeDataType type) { + this.expression = expression; + this.type = type; + } + + public H2Expression getExpression() { + return expression; + } + + public H2CompositeDataType getType() { + return type; + } + +} diff --git a/src/sqlancer/h2/ast/H2ColumnReference.java b/src/sqlancer/h2/ast/H2ColumnReference.java new file mode 100644 index 000000000..5864f0b53 --- /dev/null +++ b/src/sqlancer/h2/ast/H2ColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.h2.H2Schema; + +public class H2ColumnReference extends ColumnReferenceNode implements H2Expression { + public H2ColumnReference(H2Schema.H2Column column) { + super(column); + } + +} diff --git a/src/sqlancer/h2/H2Constant.java b/src/sqlancer/h2/ast/H2Constant.java similarity index 83% rename from src/sqlancer/h2/H2Constant.java rename to src/sqlancer/h2/ast/H2Constant.java index 4390fed26..923bcb4a2 100644 --- a/src/sqlancer/h2/H2Constant.java +++ b/src/sqlancer/h2/ast/H2Constant.java @@ -1,8 +1,6 @@ -package sqlancer.h2; +package sqlancer.h2.ast; -import sqlancer.common.ast.newast.Node; - -public class H2Constant implements Node { +public class H2Constant implements H2Expression { private H2Constant() { } @@ -108,27 +106,27 @@ public String toString() { } - public static Node createIntConstant(long val) { + public static H2Expression createIntConstant(long val) { return new H2IntConstant(val); } - public static Node createNullConstant() { + public static H2Expression createNullConstant() { return new H2NullConstant(); } - public static Node createBoolConstant(boolean val) { + public static H2Expression createBoolConstant(boolean val) { return new H2BoolConstant(val); } - public static Node createStringConstant(String val) { + public static H2Expression createStringConstant(String val) { return new H2StringConstant(val); } - public static Node createDoubleConstant(double val) { + public static H2Expression createDoubleConstant(double val) { return new H2DoubleConstant(val); } - public static Node createBinaryConstant(long val) { + public static H2Expression createBinaryConstant(long val) { return new H2BinaryConstant(val); } diff --git a/src/sqlancer/h2/H2Expression.java b/src/sqlancer/h2/ast/H2Expression.java similarity index 59% rename from src/sqlancer/h2/H2Expression.java rename to src/sqlancer/h2/ast/H2Expression.java index 61b8c9658..ed41cdfab 100644 --- a/src/sqlancer/h2/H2Expression.java +++ b/src/sqlancer/h2/ast/H2Expression.java @@ -1,4 +1,4 @@ -package sqlancer.h2; +package sqlancer.h2.ast; public interface H2Expression { diff --git a/src/sqlancer/h2/ast/H2Function.java b/src/sqlancer/h2/ast/H2Function.java new file mode 100644 index 000000000..9442d6db7 --- /dev/null +++ b/src/sqlancer/h2/ast/H2Function.java @@ -0,0 +1,11 @@ +package sqlancer.h2.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class H2Function extends NewFunctionNode implements H2Expression { + public H2Function(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/h2/ast/H2InOperation.java b/src/sqlancer/h2/ast/H2InOperation.java new file mode 100644 index 000000000..1389fb18b --- /dev/null +++ b/src/sqlancer/h2/ast/H2InOperation.java @@ -0,0 +1,11 @@ +package sqlancer.h2.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class H2InOperation extends NewInOperatorNode implements H2Expression { + public H2InOperation(H2Expression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/h2/H2Join.java b/src/sqlancer/h2/ast/H2Join.java similarity index 55% rename from src/sqlancer/h2/H2Join.java rename to src/sqlancer/h2/ast/H2Join.java index 1b566c5f0..e777e4fc9 100644 --- a/src/sqlancer/h2/H2Join.java +++ b/src/sqlancer/h2/ast/H2Join.java @@ -1,21 +1,19 @@ -package sqlancer.h2; +package sqlancer.h2.ast; import java.util.ArrayList; import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.h2.H2ExpressionGenerator; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; -import sqlancer.h2.H2Schema.H2Table; -public class H2Join implements Node { +public class H2Join implements H2Expression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final H2TableReference leftTable; + private final H2TableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final H2Expression onCondition; public enum JoinType { INNER, CROSS, NATURAL, LEFT, RIGHT; @@ -25,20 +23,19 @@ public static JoinType getRandom() { } } - public H2Join(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public H2Join(H2TableReference leftTable, H2TableReference rightTable, JoinType joinType, + H2Expression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public TableReferenceNode getLeftTable() { + public H2TableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public H2TableReference getRightTable() { return rightTable; } @@ -46,16 +43,15 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public H2Expression getOnCondition() { return onCondition; } - public static List> getJoins(List> tableList, - H2GlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, H2GlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + H2TableReference leftTable = tableList.remove(0); + H2TableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); H2ExpressionGenerator joinGen = new H2ExpressionGenerator(globalState).setColumns(columns); @@ -83,23 +79,19 @@ public static List> getJoins(List left, - TableReferenceNode right, Node predicate) { + public static H2Join createRightOuterJoin(H2TableReference left, H2TableReference right, H2Expression predicate) { return new H2Join(left, right, JoinType.RIGHT, predicate); } - public static H2Join createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static H2Join createLeftOuterJoin(H2TableReference left, H2TableReference right, H2Expression predicate) { return new H2Join(left, right, JoinType.LEFT, predicate); } - public static H2Join createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static H2Join createInnerJoin(H2TableReference left, H2TableReference right, H2Expression predicate) { return new H2Join(left, right, JoinType.INNER, predicate); } - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right) { + public static H2Expression createNaturalJoin(H2TableReference left, H2TableReference right) { return new H2Join(left, right, JoinType.NATURAL, null); } diff --git a/src/sqlancer/h2/ast/H2Select.java b/src/sqlancer/h2/ast/H2Select.java new file mode 100644 index 000000000..9658ebca0 --- /dev/null +++ b/src/sqlancer/h2/ast/H2Select.java @@ -0,0 +1,7 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.SelectBase; + +public class H2Select extends SelectBase implements H2Expression { + +} diff --git a/src/sqlancer/h2/ast/H2TableReference.java b/src/sqlancer/h2/ast/H2TableReference.java new file mode 100644 index 000000000..459337c41 --- /dev/null +++ b/src/sqlancer/h2/ast/H2TableReference.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.h2.H2Schema; + +public class H2TableReference extends TableReferenceNode implements H2Expression { + public H2TableReference(H2Schema.H2Table table) { + super(table); + } +} diff --git a/src/sqlancer/h2/ast/H2UnaryPostfixOperation.java b/src/sqlancer/h2/ast/H2UnaryPostfixOperation.java new file mode 100644 index 000000000..5c43effbe --- /dev/null +++ b/src/sqlancer/h2/ast/H2UnaryPostfixOperation.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class H2UnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements H2Expression { + public H2UnaryPostfixOperation(H2Expression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/h2/ast/H2UnaryPrefixOperation.java b/src/sqlancer/h2/ast/H2UnaryPrefixOperation.java new file mode 100644 index 000000000..24085dcb6 --- /dev/null +++ b/src/sqlancer/h2/ast/H2UnaryPrefixOperation.java @@ -0,0 +1,10 @@ +package sqlancer.h2.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class H2UnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements H2Expression { + public H2UnaryPrefixOperation(H2Expression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} From 2464a7303c6a997b5781951df3b24e493cd189d9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 10:27:55 +0800 Subject: [PATCH 474/856] [HSQLDB] Update expressions to match new AST --- .../hsqldb/HSQLDBToStringVisitor.java | 9 ++-- .../hsqldb/ast/HSQLDBBinaryOperation.java | 10 ++++ .../hsqldb/ast/HSQLDBColumnReference.java | 3 +- src/sqlancer/hsqldb/ast/HSQLDBConstant.java | 22 ++++----- src/sqlancer/hsqldb/ast/HSQLDBJoin.java | 47 +++++++++---------- src/sqlancer/hsqldb/ast/HSQLDBSelect.java | 3 +- .../hsqldb/ast/HSQLDBTableReference.java | 11 +++++ .../ast/HSQLDBUnaryPostfixOperation.java | 8 ++-- .../ast/HSQLDBUnaryPrefixOperation.java | 6 +-- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 24 +++++----- .../hsqldb/gen/HSQLDBInsertGenerator.java | 4 +- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 3 +- .../hsqldb/test/HSQLDBNoRECOracle.java | 25 +++++----- .../test/HSQLDBQueryPartitioningBase.java | 21 ++++----- 14 files changed, 99 insertions(+), 97 deletions(-) create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java create mode 100644 src/sqlancer/hsqldb/ast/HSQLDBTableReference.java diff --git a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java index 8caf7b867..99fc50687 100644 --- a/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java +++ b/src/sqlancer/hsqldb/HSQLDBToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.hsqldb; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; import sqlancer.hsqldb.ast.HSQLDBJoin; @@ -10,7 +9,7 @@ public class HSQLDBToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(HSQLDBExpression expr) { if (expr instanceof HSQLDBConstant) { visit((HSQLDBConstant) expr); } else if (expr instanceof HSQLDBSelect) { @@ -22,14 +21,14 @@ public void visitSpecific(Node expr) { } } - public static String asString(Node expr) { + public static String asString(HSQLDBExpression expr) { HSQLDBToStringVisitor visitor = new HSQLDBToStringVisitor(); visitor.visit(expr); return visitor.get(); } private void visit(HSQLDBJoin join) { - visit(join.getLeftTable()); + visit((HSQLDBExpression) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" "); @@ -37,7 +36,7 @@ private void visit(HSQLDBJoin join) { sb.append(join.getOuterType()); } sb.append(" JOIN "); - visit(join.getRightTable()); + visit((HSQLDBExpression) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); diff --git a/src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java new file mode 100644 index 000000000..b821a90e2 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class HSQLDBBinaryOperation extends NewBinaryOperatorNode implements HSQLDBExpression { + public HSQLDBBinaryOperation(HSQLDBExpression left, HSQLDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java index b2179eca3..cb579abac 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBColumnReference.java @@ -3,7 +3,8 @@ import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.hsqldb.HSQLDBSchema; -public class HSQLDBColumnReference extends ColumnReferenceNode { +public class HSQLDBColumnReference extends ColumnReferenceNode + implements HSQLDBExpression { public HSQLDBColumnReference(HSQLDBSchema.HSQLDBColumn column) { super(column); diff --git a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java index f6eed5e13..67997d6bb 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBConstant.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBConstant.java @@ -3,9 +3,7 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.common.ast.newast.Node; - -public class HSQLDBConstant implements Node { +public class HSQLDBConstant implements HSQLDBExpression { private HSQLDBConstant() { } @@ -190,39 +188,39 @@ public String toString() { } - public static Node createStringConstant(String text, int size) { + public static HSQLDBExpression createStringConstant(String text, int size) { return new HSQLDBTextConstant(text, size); } - public static Node createFloatConstant(double val) { + public static HSQLDBExpression createFloatConstant(double val) { return new HSQLDBDoubleConstant(val); } - public static Node createIntConstant(long val) { + public static HSQLDBExpression createIntConstant(long val) { return new HSQLDBIntConstant(val); } - public static Node createNullConstant() { + public static HSQLDBExpression createNullConstant() { return new HSQLDBNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static HSQLDBExpression createBooleanConstant(boolean val) { return new HSQLDBBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static HSQLDBExpression createDateConstant(long integer) { return new HSQLDBDateConstant(integer); } - public static Node createTimeConstant(long integer, int size) { + public static HSQLDBExpression createTimeConstant(long integer, int size) { return new HSQLDBTimeConstant(integer); } - public static Node createTimestampConstant(long integer, int size) { + public static HSQLDBExpression createTimestampConstant(long integer, int size) { return new HSQLDBTimestampConstant(integer); } - public static Node createBinaryConstant(long nonCachedInteger, int size) { + public static HSQLDBExpression createBinaryConstant(long nonCachedInteger, int size) { return new HSQLDBBinaryConstant(nonCachedInteger, size); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java index ef715052d..595384edd 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java @@ -4,19 +4,16 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -public class HSQLDBJoin implements Node { +public class HSQLDBJoin implements HSQLDBExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final HSQLDBTableReference leftTable; + private final HSQLDBTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final HSQLDBExpression onCondition; private OuterType outerType; public enum JoinType { @@ -35,20 +32,19 @@ public static OuterType getRandom() { } } - public HSQLDBJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public HSQLDBJoin(HSQLDBTableReference leftTable, HSQLDBTableReference rightTable, JoinType joinType, + HSQLDBExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public TableReferenceNode getLeftTable() { + public HSQLDBTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public HSQLDBTableReference getRightTable() { return rightTable; } @@ -56,7 +52,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public HSQLDBExpression getOnCondition() { return onCondition; } @@ -68,12 +64,11 @@ public OuterType getOuterType() { return outerType; } - public static List> getJoins( - List> tableList, HSQLDBGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, HSQLDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + HSQLDBTableReference leftTable = tableList.remove(0); + HSQLDBTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(globalState).setColumns(columns); @@ -100,23 +95,23 @@ public static List> getJoins( return joinExpressions; } - public static HSQLDBJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static HSQLDBJoin createRightOuterJoin(HSQLDBTableReference left, HSQLDBTableReference right, + HSQLDBExpression predicate) { return new HSQLDBJoin(left, right, JoinType.RIGHT, predicate); } - public static HSQLDBJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static HSQLDBJoin createLeftOuterJoin(HSQLDBTableReference left, HSQLDBTableReference right, + HSQLDBExpression predicate) { return new HSQLDBJoin(left, right, JoinType.LEFT, predicate); } - public static HSQLDBJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static HSQLDBJoin createInnerJoin(HSQLDBTableReference left, HSQLDBTableReference right, + HSQLDBExpression predicate) { return new HSQLDBJoin(left, right, JoinType.INNER, predicate); } - public static Node createNaturalJoin(TableReferenceNode left, - TableReferenceNode right, OuterType naturalJoinType) { + public static HSQLDBExpression createNaturalJoin(HSQLDBTableReference left, HSQLDBTableReference right, + OuterType naturalJoinType) { HSQLDBJoin join = new HSQLDBJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; diff --git a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java index 05cd390f2..c75b0b88a 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java @@ -1,9 +1,8 @@ package sqlancer.hsqldb.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class HSQLDBSelect extends SelectBase> implements Node { +public class HSQLDBSelect extends SelectBase implements HSQLDBExpression { private boolean isDistinct; diff --git a/src/sqlancer/hsqldb/ast/HSQLDBTableReference.java b/src/sqlancer/hsqldb/ast/HSQLDBTableReference.java new file mode 100644 index 000000000..010a921a7 --- /dev/null +++ b/src/sqlancer/hsqldb/ast/HSQLDBTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.hsqldb.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hsqldb.HSQLDBSchema; + +public class HSQLDBTableReference extends TableReferenceNode + implements HSQLDBExpression { + public HSQLDBTableReference(HSQLDBSchema.HSQLDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java index 9e63efee5..5ebca87fa 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPostfixOperation.java @@ -3,12 +3,12 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.hsqldb.HSQLDBSchema; -public class HSQLDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { +public class HSQLDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements HSQLDBExpression { - public HSQLDBUnaryPostfixOperation(Node expr, HSQLDBUnaryPostfixOperator op) { + public HSQLDBUnaryPostfixOperation(HSQLDBExpression expr, HSQLDBUnaryPostfixOperator op) { super(expr, op); } @@ -45,7 +45,7 @@ public String getTextRepresentation() { } - public Node getExpression() { + public HSQLDBExpression getExpression() { return getExpr(); } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java index b1327d0be..44b2beece 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBUnaryPrefixOperation.java @@ -2,12 +2,12 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.hsqldb.HSQLDBSchema; -public class HSQLDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { +public class HSQLDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements HSQLDBExpression { - public HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperator operation, Node expression) { + public HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperator operation, HSQLDBExpression expression) { super(expression, operation); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index a978dfdc3..b5304c890 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -7,11 +7,10 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.ast.HSQLDBBinaryOperation; import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; @@ -19,7 +18,7 @@ import sqlancer.hsqldb.ast.HSQLDBUnaryPrefixOperation; public final class HSQLDBExpressionGenerator extends - TypedExpressionGenerator, HSQLDBSchema.HSQLDBColumn, HSQLDBSchema.HSQLDBCompositeDataType> { + TypedExpressionGenerator { private enum Expression { BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC; @@ -32,23 +31,23 @@ public HSQLDBExpressionGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { } @Override - public Node generatePredicate() { + public HSQLDBExpression generatePredicate() { return generateExpression( HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithType(HSQLDBSchema.HSQLDBDataType.BOOLEAN)); } @Override - public Node negatePredicate(Node predicate) { + public HSQLDBExpression negatePredicate(HSQLDBExpression predicate) { return new HSQLDBUnaryPrefixOperation(HSQLDBUnaryPrefixOperation.HSQLDBUnaryPrefixOperator.NOT, predicate); } @Override - public Node isNull(Node expr) { + public HSQLDBExpression isNull(HSQLDBExpression expr) { return new HSQLDBUnaryPostfixOperation(expr, HSQLDBUnaryPostfixOperation.HSQLDBUnaryPostfixOperator.IS_NULL); } @Override - public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataType type) { + public HSQLDBExpression generateConstant(HSQLDBSchema.HSQLDBCompositeDataType type) { switch (type.getType()) { case NULL: return HSQLDBConstant.createNullConstant(); @@ -82,7 +81,7 @@ public Node generateConstant(HSQLDBSchema.HSQLDBCompositeDataT } @Override - protected Node generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { + protected HSQLDBExpression generateExpression(HSQLDBSchema.HSQLDBCompositeDataType type, int depth) { if (depth >= hsqldbGlobalState.getOptions().getMaxExpressionDepth() || Randomly.getBooleanWithSmallProbability()) { return generateLeafNode(type); @@ -105,13 +104,12 @@ protected Node generateExpression(HSQLDBSchema.HSQLDBComposite throw new AssertionError(); } - return new NewBinaryOperatorNode<>(generateExpression(type, depth + 1), generateExpression(type, depth + 1), - op); + return new HSQLDBBinaryOperation(generateExpression(type, depth + 1), generateExpression(type, depth + 1), op); } @Override - protected Node generateColumn(HSQLDBSchema.HSQLDBCompositeDataType type) { + protected HSQLDBExpression generateColumn(HSQLDBSchema.HSQLDBCompositeDataType type) { HSQLDBSchema.HSQLDBColumn column = Randomly .fromList(columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList())); return new HSQLDBColumnReference(column); @@ -183,8 +181,8 @@ public String getTextRepresentation() { } @Override - public List> generateOrderBys() { - List> expressions = new ArrayList<>(); + public List generateOrderBys() { + List expressions = new ArrayList<>(); int nr = Randomly.smallNumber() + 1; ArrayList hsqldbColumns = new ArrayList<>(columns); for (int i = 0; i < nr && !hsqldbColumns.isEmpty(); i++) { diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index c21d7c786..1cc132190 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -41,8 +40,7 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(HSQLDBSchema.HSQLDBColumn column) { - Node expression = new HSQLDBExpressionGenerator(globalState) - .generateConstant(column.getType()); + HSQLDBExpression expression = new HSQLDBExpressionGenerator(globalState).generateConstant(column.getType()); String s = HSQLDBToStringVisitor.asString(expression); sb.append(s); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index bc1a220b1..e639e21b3 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBErrors; @@ -48,7 +47,7 @@ private SQLQueryAdapter generate() { @Override protected void updateValue(HSQLDBColumn column) { - Node expr; + HSQLDBExpression expr; expr = gen.generateConstant(column.getType()); sb.append(HSQLDBToStringVisitor.asString(expr)); } diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java index eb08d061d..e5cc786c3 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -10,9 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -29,6 +26,7 @@ import sqlancer.hsqldb.ast.HSQLDBExpression; import sqlancer.hsqldb.ast.HSQLDBJoin; import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.ast.HSQLDBTableReference; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { @@ -47,12 +45,12 @@ public void check() throws SQLException { List columns = tables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList()); HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen + HSQLDBExpression randomWhereCondition = gen .generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = HSQLDBJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new HSQLDBTableReference(t)) + .collect(Collectors.toList()); + List joins = HSQLDBJoin.getJoins(tableList, state); int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); // 禁用优化 int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); if (firstCount == -1 || secondCount == -1) { @@ -64,8 +62,8 @@ public void check() throws SQLException { } } - private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + private int getSecondQuery(List tableList, HSQLDBExpression randomWhereCondition, + List joins) throws SQLException { HSQLDBSelect select = new HSQLDBSelect(); HSQLDBColumn c = new HSQLDBColumn("COUNT(*)", null, null); select.setFetchColumns(List.of(new HSQLDBColumnReference(c))); @@ -91,12 +89,11 @@ private int getSecondQuery(List> tableList, Node> tableList, - List columns, Node randomWhereCondition, List> joins) - throws SQLException { + private int getFirstQueryCount(SQLConnection con, List tableList, List columns, + HSQLDBExpression randomWhereCondition, List joins) throws SQLException { HSQLDBSelect select = new HSQLDBSelect(); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List allColumns = columns.stream().map((c) -> new HSQLDBColumnReference(c)) + .collect(Collectors.toList()); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java index 5e62a39ac..b428fd02f 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java @@ -6,22 +6,21 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBExpression; //import sqlancer.hsqldb.ast.HSQLDBJoin; import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.ast.HSQLDBTableReference; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; public class HSQLDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, HSQLDBProvider.HSQLDBGlobalState> + extends TernaryLogicPartitioningOracleBase implements TestOracle { HSQLDBSelect select; @@ -36,7 +35,7 @@ public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return expressionGenerator; } @@ -49,8 +48,7 @@ public void check() throws Exception { initializeTernaryPredicateVariants(); select = new HSQLDBSelect(); select.setFetchColumns(generateFetchColumns()); - List> tableList = targetTables.stream() - .map(t -> new TableReferenceNode(t)) + List tableList = targetTables.stream().map(t -> new HSQLDBTableReference(t)) .collect(Collectors.toList()); // List> joins = HSQLDBJoin.getJoins(tableList, state); // select.setJoinList(joins.stream().collect(Collectors.toList())); @@ -61,14 +59,13 @@ public void check() throws Exception { } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new HSQLDBSchema.HSQLDBColumn("*", null, null))); + columns.add(new HSQLDBColumnReference(new HSQLDBSchema.HSQLDBColumn("*", null, null))); } else { columns = Randomly.nonEmptySubset(targetTables.stream().flatMap(t -> t.getColumns().stream()) - .map(c -> new ColumnReferenceNode(c)) - .collect(Collectors.toList())); + .map(c -> new HSQLDBColumnReference(c)).collect(Collectors.toList())); } return columns; } From 435aff08a049765c0b8878726a8a0be995b8aa75 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 11:54:31 +0800 Subject: [PATCH 475/856] [Presto] Update expressions to match new AST --- .../presto/PrestoExpressionToNode.java | 25 -- .../presto/PrestoToStringVisitor.java | 9 +- .../presto/ast/PrestoAggregateFunction.java | 37 ++- src/sqlancer/presto/ast/PrestoAlias.java | 9 + .../presto/ast/PrestoAtTimeZoneOperator.java | 14 +- .../presto/ast/PrestoBetweenOperation.java | 10 + .../presto/ast/PrestoBinaryOperation.java | 10 + .../presto/ast/PrestoCaseOperation.java | 12 + .../presto/ast/PrestoCastFunction.java | 9 +- .../presto/ast/PrestoColumnReference.java | 3 +- src/sqlancer/presto/ast/PrestoConstant.java | 58 ++--- .../presto/ast/PrestoDefaultFunction.java | 7 +- src/sqlancer/presto/ast/PrestoFunction.java | 15 +- .../presto/ast/PrestoFunctionNode.java | 11 + .../ast/PrestoFunctionWithoutParenthesis.java | 3 +- .../presto/ast/PrestoInOperation.java | 11 + src/sqlancer/presto/ast/PrestoJoin.java | 43 ++-- .../ast/PrestoMultiValuedComparison.java | 14 +- .../presto/ast/PrestoPostfixText.java | 9 + .../ast/PrestoQuantifiedComparison.java | 14 +- src/sqlancer/presto/ast/PrestoSelect.java | 3 +- .../presto/ast/PrestoTableReference.java | 3 +- src/sqlancer/presto/ast/PrestoTernary.java | 10 + .../ast/PrestoUnaryPostfixOperation.java | 8 +- .../ast/PrestoUnaryPrefixOperation.java | 6 +- .../presto/gen/PrestoIndexGenerator.java | 3 +- .../presto/gen/PrestoInsertGenerator.java | 3 +- .../gen/PrestoRandomQuerySynthesizer.java | 13 +- .../gen/PrestoTypedExpressionGenerator.java | 239 +++++++++--------- .../presto/gen/PrestoUpdateGenerator.java | 3 +- .../presto/test/PrestoNoRECOracle.java | 32 ++- ...restoQueryPartitioningAggregateTester.java | 41 ++- .../test/PrestoQueryPartitioningBase.java | 26 +- .../PrestoQueryPartitioningGroupByTester.java | 10 +- .../PrestoQueryPartitioningHavingTester.java | 5 +- .../ycql/ast/YCQLBetweenOperation.java | 10 + .../ycql/ast/YCQLBinaryOperation.java | 11 + .../ycql/ast/YCQLColumnReference.java | 13 + .../yugabyte/ycql/ast/YCQLFunction.java | 12 + .../yugabyte/ycql/ast/YCQLInOperation.java | 12 + .../yugabyte/ycql/ast/YCQLOrderingTerm.java | 10 + .../yugabyte/ycql/ast/YCQLTableReference.java | 12 + .../ycql/ast/YCQLUnaryPostfixOperation.java | 12 + .../ycql/ast/YCQLUnaryPrefixOperation.java | 12 + 44 files changed, 478 insertions(+), 354 deletions(-) delete mode 100644 src/sqlancer/presto/PrestoExpressionToNode.java create mode 100644 src/sqlancer/presto/ast/PrestoAlias.java create mode 100644 src/sqlancer/presto/ast/PrestoBetweenOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoBinaryOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoCaseOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoFunctionNode.java create mode 100644 src/sqlancer/presto/ast/PrestoInOperation.java create mode 100644 src/sqlancer/presto/ast/PrestoPostfixText.java create mode 100644 src/sqlancer/presto/ast/PrestoTernary.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java create mode 100644 src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java diff --git a/src/sqlancer/presto/PrestoExpressionToNode.java b/src/sqlancer/presto/PrestoExpressionToNode.java deleted file mode 100644 index 96497e47c..000000000 --- a/src/sqlancer/presto/PrestoExpressionToNode.java +++ /dev/null @@ -1,25 +0,0 @@ -package sqlancer.presto; - -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.ast.newast.Node; -import sqlancer.presto.ast.PrestoExpression; - -public final class PrestoExpressionToNode { - - private PrestoExpressionToNode() { - - } - - @SuppressWarnings("unchecked") - public static Node cast(PrestoExpression expression) { - return (Node) expression; - } - - @SuppressWarnings("unchecked") - public static List> casts(List expressions) { - return expressions.stream().map(e -> (Node) e).collect(Collectors.toList()); - } - -} diff --git a/src/sqlancer/presto/PrestoToStringVisitor.java b/src/sqlancer/presto/PrestoToStringVisitor.java index f840cf850..11c6d5fa6 100644 --- a/src/sqlancer/presto/PrestoToStringVisitor.java +++ b/src/sqlancer/presto/PrestoToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.presto; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.ast.PrestoAtTimeZoneOperator; import sqlancer.presto.ast.PrestoCastFunction; import sqlancer.presto.ast.PrestoConstant; @@ -14,14 +13,14 @@ public class PrestoToStringVisitor extends NewToStringVisitor { - public static String asString(Node expr) { + public static String asString(PrestoExpression expr) { PrestoToStringVisitor visitor = new PrestoToStringVisitor(); visitor.visit(expr); return visitor.get(); } @Override - public void visitSpecific(Node expr) { + public void visitSpecific(PrestoExpression expr) { if (expr instanceof PrestoConstant) { visit((PrestoConstant) expr); } else if (expr instanceof PrestoSelect) { @@ -44,7 +43,7 @@ public void visitSpecific(Node expr) { } private void visit(PrestoJoin join) { - visit(join.getLeftTable()); + visit((PrestoExpression) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" "); @@ -52,7 +51,7 @@ private void visit(PrestoJoin join) { sb.append(join.getOuterType()); } sb.append(" JOIN "); - visit(join.getRightTable()); + visit((PrestoExpression) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); diff --git a/src/sqlancer/presto/ast/PrestoAggregateFunction.java b/src/sqlancer/presto/ast/PrestoAggregateFunction.java index 0b118393f..25840933a 100644 --- a/src/sqlancer/presto/ast/PrestoAggregateFunction.java +++ b/src/sqlancer/presto/ast/PrestoAggregateFunction.java @@ -8,7 +8,6 @@ import java.util.stream.Stream; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; import sqlancer.presto.PrestoSchema.PrestoDataType; @@ -37,7 +36,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -131,7 +130,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -165,7 +164,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -211,7 +210,7 @@ public PrestoCompositeDataType getCompositeReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -248,7 +247,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -338,7 +337,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -427,9 +426,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); arguments.add(gen.generateExpression(returnType2, depth + 1)); arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); return arguments; @@ -441,7 +440,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType.fromDataType(getReturnType())); @@ -467,9 +466,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); arguments.add(gen.generateExpression(returnType2, depth + 1)); arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); if (Randomly.getBooleanWithRatherLowProbability()) { @@ -486,7 +485,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType @@ -523,9 +522,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); arguments.add(gen.generateExpression(returnType2, depth + 1)); arguments.add(new PrestoConstant.PrestoFloatConstant(Randomly.getPercentage())); if (Randomly.getBooleanWithRatherLowProbability()) { @@ -542,7 +541,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType @@ -566,9 +565,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes2, PrestoCompositeDataType returnType2) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); arguments.add(gen.generateExpression(returnType2, depth + 1)); if (Randomly.getBooleanWithRatherLowProbability()) { arguments.add(new PrestoConstant.PrestoIntConstant(1)); @@ -590,7 +589,7 @@ public PrestoDataType getReturnType() { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoCompositeDataType returnType, boolean orderable) { PrestoCompositeDataType returnTypeLocal = Objects.requireNonNullElseGet(returnType, () -> PrestoCompositeDataType diff --git a/src/sqlancer/presto/ast/PrestoAlias.java b/src/sqlancer/presto/ast/PrestoAlias.java new file mode 100644 index 000000000..b327bed11 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoAlias.java @@ -0,0 +1,9 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewAliasNode; + +public class PrestoAlias extends NewAliasNode implements PrestoExpression { + public PrestoAlias(PrestoExpression expr, String alias) { + super(expr, alias); + } +} diff --git a/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java index 7bb01bb98..ac73a0a79 100644 --- a/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java +++ b/src/sqlancer/presto/ast/PrestoAtTimeZoneOperator.java @@ -1,22 +1,20 @@ package sqlancer.presto.ast; -import sqlancer.common.ast.newast.Node; +public class PrestoAtTimeZoneOperator implements PrestoExpression { -public class PrestoAtTimeZoneOperator implements Node { + private final PrestoExpression expr; + private final PrestoExpression timeZone; - private final Node expr; - private final Node timeZone; - - public PrestoAtTimeZoneOperator(Node expr, Node timeZone) { + public PrestoAtTimeZoneOperator(PrestoExpression expr, PrestoExpression timeZone) { this.expr = expr; this.timeZone = timeZone; } - public Node getExpr() { + public PrestoExpression getExpr() { return expr; } - public Node getTimeZone() { + public PrestoExpression getTimeZone() { return timeZone; } } diff --git a/src/sqlancer/presto/ast/PrestoBetweenOperation.java b/src/sqlancer/presto/ast/PrestoBetweenOperation.java new file mode 100644 index 000000000..394a5f675 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class PrestoBetweenOperation extends NewBetweenOperatorNode implements PrestoExpression { + public PrestoBetweenOperation(PrestoExpression left, PrestoExpression middle, PrestoExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/presto/ast/PrestoBinaryOperation.java b/src/sqlancer/presto/ast/PrestoBinaryOperation.java new file mode 100644 index 000000000..90f83f31a --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class PrestoBinaryOperation extends NewBinaryOperatorNode implements PrestoExpression { + public PrestoBinaryOperation(PrestoExpression left, PrestoExpression right, BinaryOperatorNode.Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/presto/ast/PrestoCaseOperation.java b/src/sqlancer/presto/ast/PrestoCaseOperation.java new file mode 100644 index 000000000..117c54bbb --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoCaseOperation.java @@ -0,0 +1,12 @@ +package sqlancer.presto.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class PrestoCaseOperation extends NewCaseOperatorNode implements PrestoExpression { + public PrestoCaseOperation(PrestoExpression switchCondition, List conditions, + List expressions, PrestoExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/presto/ast/PrestoCastFunction.java b/src/sqlancer/presto/ast/PrestoCastFunction.java index 89b139314..14798267d 100644 --- a/src/sqlancer/presto/ast/PrestoCastFunction.java +++ b/src/sqlancer/presto/ast/PrestoCastFunction.java @@ -1,19 +1,18 @@ package sqlancer.presto.ast; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; -public class PrestoCastFunction implements Node { +public class PrestoCastFunction implements PrestoExpression { - private final Node expr; + private final PrestoExpression expr; private final PrestoSchema.PrestoCompositeDataType type; - public PrestoCastFunction(Node expr, PrestoSchema.PrestoCompositeDataType type) { + public PrestoCastFunction(PrestoExpression expr, PrestoSchema.PrestoCompositeDataType type) { this.expr = expr; this.type = type; } - public Node getExpr() { + public PrestoExpression getExpr() { return expr; } diff --git a/src/sqlancer/presto/ast/PrestoColumnReference.java b/src/sqlancer/presto/ast/PrestoColumnReference.java index 4f96a013d..8bc6db0ec 100644 --- a/src/sqlancer/presto/ast/PrestoColumnReference.java +++ b/src/sqlancer/presto/ast/PrestoColumnReference.java @@ -3,7 +3,8 @@ import sqlancer.common.ast.newast.ColumnReferenceNode; import sqlancer.presto.PrestoSchema; -public class PrestoColumnReference extends ColumnReferenceNode { +public class PrestoColumnReference extends ColumnReferenceNode + implements PrestoExpression { public PrestoColumnReference(PrestoSchema.PrestoColumn column) { super(column); diff --git a/src/sqlancer/presto/ast/PrestoConstant.java b/src/sqlancer/presto/ast/PrestoConstant.java index 97b115a4e..b1d30a1ab 100644 --- a/src/sqlancer/presto/ast/PrestoConstant.java +++ b/src/sqlancer/presto/ast/PrestoConstant.java @@ -9,11 +9,10 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoConstantUtils; import sqlancer.presto.PrestoSchema; -public abstract class PrestoConstant implements Node, PrestoExpression { +public abstract class PrestoConstant implements PrestoExpression { private static final String[] TIME_ZONES = { "Africa/Abidjan", "Africa/Accra", "Africa/Addis_Ababa", "Africa/Algiers", "Africa/Asmara", "Africa/Asmera", "Africa/Bamako", "Africa/Bangui", "Africa/Banjul", @@ -127,44 +126,44 @@ public abstract class PrestoConstant implements Node, PrestoEx private PrestoConstant() { } - public static Node createStringConstant(String text) { + public static PrestoExpression createStringConstant(String text) { return new PrestoTextConstant(text); } - public static Node createStringConstant(String text, int size) { + public static PrestoExpression createStringConstant(String text, int size) { return new PrestoTextConstant(text, size); } - public static Node createJsonConstant() { + public static PrestoExpression createJsonConstant() { return new PrestoJsonConstant(); } - public static Node createFloatConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + public static PrestoExpression createFloatConstant(PrestoSchema.PrestoCompositeDataType type, double val) { assert type.getSize() == 4; float floatValue = (float) val; return new PrestoFloatConstant(floatValue); } - public static Node createFloatConstant(double val) { + public static PrestoExpression createFloatConstant(double val) { return new PrestoFloatConstant(val); } - public static Node createDecimalConstant(double val) { + public static PrestoExpression createDecimalConstant(double val) { return new PrestoDecimalConstant(val); } - public static Node createDecimalConstant(PrestoSchema.PrestoCompositeDataType type, double val) { + public static PrestoExpression createDecimalConstant(PrestoSchema.PrestoCompositeDataType type, double val) { int scale = type.getScale(); int precision = type.getSize(); BigDecimal finalBD = PrestoConstantUtils.getDecimal(val, scale, precision); return new PrestoDecimalConstant(finalBD.doubleValue()); } - public static Node createIntConstant(long val) { + public static PrestoExpression createIntConstant(long val) { return new PrestoIntConstant(val); } - public static Node createIntConstant(PrestoSchema.PrestoCompositeDataType type, long val, + public static PrestoExpression createIntConstant(PrestoSchema.PrestoCompositeDataType type, long val, boolean castInteger) { PrestoIntConstant intConstant; assert List.of(1, 2, 4, 8).contains(type.getSize()); @@ -188,56 +187,56 @@ public static Node createIntConstant(PrestoSchema.PrestoCompos } } - public static Node createNullConstant() { + public static PrestoExpression createNullConstant() { return new PrestoNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static PrestoExpression createBooleanConstant(boolean val) { return new PrestoBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static PrestoExpression createDateConstant(long integer) { return new PrestoDateConstant(integer); } - public static Node createTimeConstant(long integer) { + public static PrestoExpression createTimeConstant(long integer) { return new PrestoTimeConstant(integer); } - public static Node createTimeWithTimeZoneConstant(long integer) { + public static PrestoExpression createTimeWithTimeZoneConstant(long integer) { return new PrestoTimeWithTimeZoneConstant(integer); } - public static Node createTimestampWithTimeZoneConstant(long integer) { + public static PrestoExpression createTimestampWithTimeZoneConstant(long integer) { return new PrestoTimestampWithTimezoneConstant(integer); } - public static Node createIntervalDayToSecond(long integer) { + public static PrestoExpression createIntervalDayToSecond(long integer) { return new PrestoIntervalDayToSecondConstant(); } - public static Node createIntervalYearToMonth(long integer) { + public static PrestoExpression createIntervalYearToMonth(long integer) { return new PrestoIntervalYearToMonthConstant(); } - public static Node createTimestampConstant(long integer) { + public static PrestoExpression createTimestampConstant(long integer) { return new PrestoTimestampConstant(integer); } - public static Node createVarbinaryConstant(String string) { + public static PrestoExpression createVarbinaryConstant(String string) { return new PrestoVarbinaryConstant(string); } - public static Node createTimezoneConstant() { + public static PrestoExpression createTimezoneConstant() { String string = Randomly.fromOptions(TIME_ZONES); return new PrestoTextConstant(string); } - public static Node createArrayConstant(PrestoSchema.PrestoCompositeDataType type) { + public static PrestoExpression createArrayConstant(PrestoSchema.PrestoCompositeDataType type) { PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); long size = Randomly.getNotCachedInteger(0, 10); - List> elements = new ArrayList<>(); + List elements = new ArrayList<>(); for (int i = 0; i <= size; i++) { if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { elements.add(createArrayConstant(elementType)); @@ -248,11 +247,11 @@ public static Node createArrayConstant(PrestoSchema.PrestoComp return new PrestoArrayConstant(elements); } - public static Node createMapConstant(PrestoSchema.PrestoCompositeDataType type) { + public static PrestoExpression createMapConstant(PrestoSchema.PrestoCompositeDataType type) { PrestoSchema.PrestoCompositeDataType elementType = type.getElementType(); long size = Randomly.getNotCachedInteger(0, 10); - List> elements = new ArrayList<>(); + List elements = new ArrayList<>(); for (int i = 0; i <= size; i++) { if (elementType.getPrimitiveDataType() == PrestoSchema.PrestoDataType.ARRAY) { elements.add(createArrayConstant(elementType)); @@ -263,8 +262,7 @@ public static Node createMapConstant(PrestoSchema.PrestoCompos return new PrestoArrayConstant(elements); } - public static Node generateConstant(PrestoSchema.PrestoCompositeDataType type, - boolean castInteger) { + public static PrestoExpression generateConstant(PrestoSchema.PrestoCompositeDataType type, boolean castInteger) { Randomly randomly = new Randomly(); switch (type.getPrimitiveDataType()) { case ARRAY: @@ -789,9 +787,9 @@ public boolean isBoolean() { public static class PrestoArrayConstant extends PrestoConstant { - private final List> elements; + private final List elements; - public PrestoArrayConstant(List> elements) { + public PrestoArrayConstant(List elements) { this.elements = new ArrayList<>(elements); } diff --git a/src/sqlancer/presto/ast/PrestoDefaultFunction.java b/src/sqlancer/presto/ast/PrestoDefaultFunction.java index 95b89091e..99b7f7fde 100644 --- a/src/sqlancer/presto/ast/PrestoDefaultFunction.java +++ b/src/sqlancer/presto/ast/PrestoDefaultFunction.java @@ -6,7 +6,6 @@ import java.util.stream.Stream; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; import sqlancer.presto.PrestoSchema.PrestoDataType; @@ -75,7 +74,7 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { return super.getArgumentsForReturnType(gen, depth, argumentTypes, returnType); } @@ -177,9 +176,9 @@ public PrestoDataType[] getArgumentTypes(PrestoCompositeDataType returnType) { } @Override - public List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + public List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoDataType[] argumentTypes, PrestoCompositeDataType returnType) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); // This is a workaround based on the assumption that array types should refer to the same element type. PrestoCompositeDataType savedArrayType = null; diff --git a/src/sqlancer/presto/ast/PrestoFunction.java b/src/sqlancer/presto/ast/PrestoFunction.java index c91759e2f..fc7ef5b79 100644 --- a/src/sqlancer/presto/ast/PrestoFunction.java +++ b/src/sqlancer/presto/ast/PrestoFunction.java @@ -4,11 +4,10 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public interface PrestoFunction { +public interface PrestoFunction extends PrestoExpression { String getFunctionName(); @@ -16,10 +15,10 @@ public interface PrestoFunction { PrestoSchema.PrestoDataType[] getArgumentTypes(PrestoSchema.PrestoCompositeDataType returnType); - default List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + default List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoSchema.PrestoDataType[] argumentTypes, PrestoSchema.PrestoCompositeDataType returnType) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); // This is a workaround based on the assumption that array types should refer to // the same element type. @@ -56,17 +55,17 @@ default List> getArgumentsForReturnType(PrestoTypedExpres } else { dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); } - Node expression = gen.generateExpression(dataType, depth + 1); + PrestoExpression expression = gen.generateExpression(dataType, depth + 1); arguments.add(expression); } } return arguments; } - default List> getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, + default List getArgumentsForReturnType(PrestoTypedExpressionGenerator gen, int depth, PrestoSchema.PrestoCompositeDataType returnType, boolean orderable) { - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); // This is a workaround based on the assumption that array types should refer to // the same element type. @@ -105,7 +104,7 @@ default List> getArgumentsForReturnType(PrestoTypedExpres } else { compositeDataType = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); } - Node expression = gen.generateExpression(compositeDataType, depth + 1); + PrestoExpression expression = gen.generateExpression(compositeDataType, depth + 1); arguments.add(expression); } } diff --git a/src/sqlancer/presto/ast/PrestoFunctionNode.java b/src/sqlancer/presto/ast/PrestoFunctionNode.java new file mode 100644 index 000000000..14409a825 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoFunctionNode.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class PrestoFunctionNode extends NewFunctionNode implements PrestoExpression { + public PrestoFunctionNode(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java index 5b184671b..88007053d 100644 --- a/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java +++ b/src/sqlancer/presto/ast/PrestoFunctionWithoutParenthesis.java @@ -1,9 +1,8 @@ package sqlancer.presto.ast; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; -public class PrestoFunctionWithoutParenthesis implements Node { +public class PrestoFunctionWithoutParenthesis implements PrestoExpression { private final PrestoSchema.PrestoCompositeDataType type; private final String expr; diff --git a/src/sqlancer/presto/ast/PrestoInOperation.java b/src/sqlancer/presto/ast/PrestoInOperation.java new file mode 100644 index 000000000..95edd7fc3 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoInOperation.java @@ -0,0 +1,11 @@ +package sqlancer.presto.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class PrestoInOperation extends NewInOperatorNode implements PrestoExpression { + public PrestoInOperation(PrestoExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/presto/ast/PrestoJoin.java b/src/sqlancer/presto/ast/PrestoJoin.java index 79bf7e1fc..0732e55be 100644 --- a/src/sqlancer/presto/ast/PrestoJoin.java +++ b/src/sqlancer/presto/ast/PrestoJoin.java @@ -4,37 +4,32 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoColumn; -import sqlancer.presto.PrestoSchema.PrestoTable; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoJoin implements Node { +public class PrestoJoin implements PrestoExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final PrestoTableReference leftTable; + private final PrestoTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final PrestoExpression onCondition; private OuterType outerType; - public PrestoJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public PrestoJoin(PrestoTableReference leftTable, PrestoTableReference rightTable, JoinType joinType, + PrestoExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public static List> getJoins( - List> tableList, PrestoGlobalState globalState) { - List> joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, PrestoGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + PrestoTableReference leftTable = tableList.remove(0); + PrestoTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); PrestoTypedExpressionGenerator joinGen = new PrestoTypedExpressionGenerator(globalState) @@ -59,26 +54,26 @@ public static List> getJoins( return joinExpressions; } - public static PrestoJoin createRightOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static PrestoJoin createRightOuterJoin(PrestoTableReference left, PrestoTableReference right, + PrestoExpression predicate) { return new PrestoJoin(left, right, JoinType.RIGHT, predicate); } - public static PrestoJoin createLeftOuterJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static PrestoJoin createLeftOuterJoin(PrestoTableReference left, PrestoTableReference right, + PrestoExpression predicate) { return new PrestoJoin(left, right, JoinType.LEFT, predicate); } - public static PrestoJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static PrestoJoin createInnerJoin(PrestoTableReference left, PrestoTableReference right, + PrestoExpression predicate) { return new PrestoJoin(left, right, JoinType.INNER, predicate); } - public TableReferenceNode getLeftTable() { + public PrestoTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public PrestoTableReference getRightTable() { return rightTable; } @@ -86,7 +81,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public PrestoExpression getOnCondition() { return onCondition; } diff --git a/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java index 47fc209fb..20ceaee79 100644 --- a/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java +++ b/src/sqlancer/presto/ast/PrestoMultiValuedComparison.java @@ -3,16 +3,14 @@ import java.util.ArrayList; import java.util.List; -import sqlancer.common.ast.newast.Node; +public class PrestoMultiValuedComparison implements PrestoExpression { -public class PrestoMultiValuedComparison implements Node { - - private final Node left; - private final List> right; + private final PrestoExpression left; + private final List right; private final PrestoMultiValuedComparisonType type; private final PrestoMultiValuedComparisonOperator op; - public PrestoMultiValuedComparison(Node left, List> right, + public PrestoMultiValuedComparison(PrestoExpression left, List right, PrestoMultiValuedComparisonType type, PrestoMultiValuedComparisonOperator op) { this.left = left; this.right = new ArrayList<>(right); @@ -20,7 +18,7 @@ public PrestoMultiValuedComparison(Node left, List getLeft() { + public PrestoExpression getLeft() { return left; } @@ -28,7 +26,7 @@ public PrestoMultiValuedComparisonOperator getOp() { return op; } - public List> getRight() { + public List getRight() { return new ArrayList<>(right); } diff --git a/src/sqlancer/presto/ast/PrestoPostfixText.java b/src/sqlancer/presto/ast/PrestoPostfixText.java new file mode 100644 index 000000000..de99fede4 --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoPostfixText.java @@ -0,0 +1,9 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewPostfixTextNode; + +public class PrestoPostfixText extends NewPostfixTextNode implements PrestoExpression { + public PrestoPostfixText(PrestoExpression expr, String text) { + super(expr, text); + } +} diff --git a/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java index d51539645..3cbccc7b2 100644 --- a/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java +++ b/src/sqlancer/presto/ast/PrestoQuantifiedComparison.java @@ -1,23 +1,21 @@ package sqlancer.presto.ast; -import sqlancer.common.ast.newast.Node; +public class PrestoQuantifiedComparison implements PrestoExpression { -public class PrestoQuantifiedComparison implements Node { - - private final Node left; + private final PrestoExpression left; private final PrestoSelect right; private final PrestoMultiValuedComparisonType type; private final PrestoMultiValuedComparisonOperator op; - public PrestoQuantifiedComparison(Node left, PrestoSelect right, - PrestoMultiValuedComparisonType type, PrestoMultiValuedComparisonOperator op) { + public PrestoQuantifiedComparison(PrestoExpression left, PrestoSelect right, PrestoMultiValuedComparisonType type, + PrestoMultiValuedComparisonOperator op) { this.left = left; this.right = right; this.type = type; this.op = op; } - public Node getLeft() { + public PrestoExpression getLeft() { return left; } @@ -25,7 +23,7 @@ public PrestoMultiValuedComparisonOperator getOp() { return op; } - public Node getRight() { + public PrestoExpression getRight() { return right; } diff --git a/src/sqlancer/presto/ast/PrestoSelect.java b/src/sqlancer/presto/ast/PrestoSelect.java index c99c5c7d0..328c154c4 100644 --- a/src/sqlancer/presto/ast/PrestoSelect.java +++ b/src/sqlancer/presto/ast/PrestoSelect.java @@ -1,9 +1,8 @@ package sqlancer.presto.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class PrestoSelect extends SelectBase> implements Node { +public class PrestoSelect extends SelectBase implements PrestoExpression { private boolean isDistinct; diff --git a/src/sqlancer/presto/ast/PrestoTableReference.java b/src/sqlancer/presto/ast/PrestoTableReference.java index 4c0e0d789..7552ecc64 100644 --- a/src/sqlancer/presto/ast/PrestoTableReference.java +++ b/src/sqlancer/presto/ast/PrestoTableReference.java @@ -3,7 +3,8 @@ import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.presto.PrestoSchema; -public class PrestoTableReference extends TableReferenceNode { +public class PrestoTableReference extends TableReferenceNode + implements PrestoExpression { public PrestoTableReference(PrestoSchema.PrestoTable table) { super(table); diff --git a/src/sqlancer/presto/ast/PrestoTernary.java b/src/sqlancer/presto/ast/PrestoTernary.java new file mode 100644 index 000000000..618daa0fc --- /dev/null +++ b/src/sqlancer/presto/ast/PrestoTernary.java @@ -0,0 +1,10 @@ +package sqlancer.presto.ast; + +import sqlancer.common.ast.newast.NewTernaryNode; + +public class PrestoTernary extends NewTernaryNode implements PrestoExpression { + public PrestoTernary(PrestoExpression left, PrestoExpression middle, PrestoExpression right, String leftStr, + String rightStr) { + super(left, middle, right, leftStr, rightStr); + } +} diff --git a/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java index 2bdc142e6..99677d3f8 100644 --- a/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java +++ b/src/sqlancer/presto/ast/PrestoUnaryPostfixOperation.java @@ -3,16 +3,16 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; -public class PrestoUnaryPostfixOperation extends NewUnaryPostfixOperatorNode { +public class PrestoUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements PrestoExpression { - public PrestoUnaryPostfixOperation(Node expr, PrestoUnaryPostfixOperator op) { + public PrestoUnaryPostfixOperation(PrestoExpression expr, PrestoUnaryPostfixOperator op) { super(expr, op); } - public Node getExpression() { + public PrestoExpression getExpression() { return getExpr(); } diff --git a/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java index cb8313622..db7a08068 100644 --- a/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java +++ b/src/sqlancer/presto/ast/PrestoUnaryPrefixOperation.java @@ -3,12 +3,12 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoSchema; -public class PrestoUnaryPrefixOperation extends NewUnaryPrefixOperatorNode { +public class PrestoUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements PrestoExpression { - public PrestoUnaryPrefixOperation(PrestoUnaryPrefixOperator operation, Node expression) { + public PrestoUnaryPrefixOperation(PrestoExpression expression, BinaryOperatorNode.Operator operation) { super(expression, operation); } diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java index 7a467e37d..c76283ee4 100644 --- a/src/sqlancer/presto/gen/PrestoIndexGenerator.java +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoGlobalState; @@ -46,7 +45,7 @@ public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { sb.append(")"); if (Randomly.getBoolean()) { sb.append(" WHERE "); - Node expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) + PrestoExpression expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); sb.append(PrestoToStringVisitor.asString(expr)); } diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index 102d2f1be..81e50e1d1 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import java.util.stream.Collectors; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractInsertGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -43,7 +42,7 @@ private SQLQueryAdapter generate() { @Override protected void insertValue(PrestoColumn prestoColumn) { - Node constant = new PrestoTypedExpressionGenerator(globalState) + PrestoExpression constant = new PrestoTypedExpressionGenerator(globalState) .generateInsertConstant(prestoColumn.getType()); sb.append(PrestoToStringVisitor.asString(constant)); diff --git a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java index 2d18311b6..27a54d926 100644 --- a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java +++ b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java @@ -5,8 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoTable; @@ -15,6 +13,7 @@ import sqlancer.presto.ast.PrestoExpression; import sqlancer.presto.ast.PrestoJoin; import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; public final class PrestoRandomQuerySynthesizer { @@ -29,10 +28,10 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC // TODO: distinct // select.setDistinct(Randomly.getBoolean()); // boolean allowAggregates = Randomly.getBooleanWithSmallProbability(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { // if (allowAggregates && Randomly.getBoolean()) { - Node expression = gen + PrestoExpression expression = gen .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); columns.add(expression); // } else { @@ -41,9 +40,9 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC } select.setFetchColumns(columns); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = PrestoJoin.getJoins(tableList, globalState); + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, globalState); select.setJoinList(new ArrayList<>(joins)); select.setFromList(new ArrayList<>(tableList)); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java index 79e657fda..712ebdcc3 100644 --- a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -9,38 +9,35 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewCaseOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewTernaryNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.ast.PrestoAggregateFunction; import sqlancer.presto.ast.PrestoAtTimeZoneOperator; +import sqlancer.presto.ast.PrestoBetweenOperation; +import sqlancer.presto.ast.PrestoBinaryOperation; +import sqlancer.presto.ast.PrestoCaseOperation; import sqlancer.presto.ast.PrestoCastFunction; import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoConstant; import sqlancer.presto.ast.PrestoDefaultFunction; import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoFunctionNode; +import sqlancer.presto.ast.PrestoInOperation; import sqlancer.presto.ast.PrestoJoin; import sqlancer.presto.ast.PrestoMultiValuedComparison; import sqlancer.presto.ast.PrestoMultiValuedComparisonOperator; import sqlancer.presto.ast.PrestoMultiValuedComparisonType; import sqlancer.presto.ast.PrestoQuantifiedComparison; import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; +import sqlancer.presto.ast.PrestoTernary; import sqlancer.presto.ast.PrestoUnaryPostfixOperation; import sqlancer.presto.ast.PrestoUnaryPrefixOperation; public final class PrestoTypedExpressionGenerator extends - TypedExpressionGenerator, PrestoSchema.PrestoColumn, PrestoSchema.PrestoCompositeDataType> { + TypedExpressionGenerator { private final Randomly randomly; private final PrestoGlobalState globalState; @@ -53,24 +50,24 @@ public PrestoTypedExpressionGenerator(PrestoGlobalState globalState) { } @Override - public Node generatePredicate() { + public PrestoExpression generatePredicate() { return generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), randomly.getInteger(0, maxDepth)); } @Override - public Node negatePredicate(Node predicate) { - return new PrestoUnaryPrefixOperation(PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT, predicate); + public PrestoExpression negatePredicate(PrestoExpression predicate) { + return new PrestoUnaryPrefixOperation(predicate, PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { + public PrestoExpression isNull(PrestoExpression expr) { return new PrestoUnaryPostfixOperation(expr, PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); } @Override - public Node generateConstant(PrestoSchema.PrestoCompositeDataType type) { + public PrestoExpression generateConstant(PrestoSchema.PrestoCompositeDataType type) { if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { return PrestoConstant.createArrayConstant(type); // case MAP: @@ -79,7 +76,7 @@ public Node generateConstant(PrestoSchema.PrestoCompositeDataT return PrestoConstant.generateConstant(type, false); } - public Node generateInsertConstant(PrestoSchema.PrestoCompositeDataType type) { + public PrestoExpression generateInsertConstant(PrestoSchema.PrestoCompositeDataType type) { if (Objects.requireNonNull(type.getPrimitiveDataType()) == PrestoSchema.PrestoDataType.ARRAY) { return PrestoConstant.createArrayConstant(type); // case MAP: @@ -89,7 +86,7 @@ public Node generateInsertConstant(PrestoSchema.PrestoComposit } @Override - public Node generateExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + public PrestoExpression generateExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { if (allowAggregates && Randomly.getBoolean()) { return generateAggregate(type); } @@ -146,52 +143,52 @@ public Node generateExpression(PrestoSchema.PrestoCompositeDat } } - private Node generateJsonExpression(PrestoSchema.PrestoCompositeDataType type) { + private PrestoExpression generateJsonExpression(PrestoSchema.PrestoCompositeDataType type) { return generateLeafNode(type); } - private Node generateCast(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateCast(PrestoSchema.PrestoCompositeDataType type, int depth) { // check can cast - Node expressionNode = generateExpression(getRandomType(), depth + 1); + PrestoExpression expressionNode = generateExpression(getRandomType(), depth + 1); return new PrestoCastFunction(expressionNode, type); } @SuppressWarnings("unused") - private Node generateTry(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateTry(PrestoSchema.PrestoCompositeDataType type, int depth) { if (type.getPrimitiveDataType().isNumeric() && Randomly.getBooleanWithRatherLowProbability()) { - Node expression = generateExpression(type); - return new NewFunctionNode<>(List.of(expression), "try"); + PrestoExpression expression = generateExpression(type); + return new PrestoFunctionNode<>(List.of(expression), "try"); } List applicableFunctions = PrestoDefaultFunction.getFunctionsCompatibleWith(type); if (Randomly.getBooleanWithRatherLowProbability() && !applicableFunctions.isEmpty()) { PrestoDefaultFunction function = Randomly.fromList(applicableFunctions); - Node expression = generateFunction(type, depth, function); - return new NewFunctionNode<>(List.of(expression), "try"); + PrestoExpression expression = generateFunction(type, depth, function); + return new PrestoFunctionNode<>(List.of(expression), "try"); } - return new NewFunctionNode<>(List.of(generateCast(type, depth)), "try"); + return new PrestoFunctionNode<>(List.of(generateCast(type, depth)), "try"); } - private NewCaseOperatorNode getCase(PrestoSchema.PrestoCompositeDataType type, int depth) { - List> conditions = new ArrayList<>(); - List> cases = new ArrayList<>(); + private PrestoCaseOperation getCase(PrestoSchema.PrestoCompositeDataType type, int depth) { + List conditions = new ArrayList<>(); + List cases = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { conditions.add(generateExpression(type, depth + 1)); cases.add(generateExpression(type, depth + 1)); } - Node elseExpr = null; + PrestoExpression elseExpr = null; if (Randomly.getBoolean()) { elseExpr = generateExpression(type, depth + 1); } - Node expression = generateExpression(type); - return new NewCaseOperatorNode<>(expression, conditions, cases, elseExpr); + PrestoExpression expression = generateExpression(type); + return new PrestoCaseOperation(expression, conditions, cases, elseExpr); } - private Node generateFunction(PrestoSchema.PrestoCompositeDataType returnType, int depth, + private PrestoExpression generateFunction(PrestoSchema.PrestoCompositeDataType returnType, int depth, PrestoDefaultFunction function) { PrestoSchema.PrestoDataType[] argumentTypes = function.getArgumentTypes(returnType); - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); // This is a workaround based on the assumption that array types should refer to // the same element type. @@ -227,34 +224,34 @@ private Node generateFunction(PrestoSchema.PrestoCompositeData } else { dataType = PrestoSchema.PrestoCompositeDataType.fromDataType(arg); } - Node expression = generateExpression(dataType, depth + 1); + PrestoExpression expression = generateExpression(dataType, depth + 1); arguments.add(expression); } } - return new NewFunctionNode<>(arguments, function); + return new PrestoFunctionNode<>(arguments, function); } - private Node generateStringExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateStringExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(type); } return getStringOperation(depth); } - private NewBinaryOperatorNode getStringOperation(int depth) { + private PrestoBinaryOperation getStringOperation(int depth) { StringExpression exprType = Randomly.fromOptions(StringExpression.values()); if (Objects.requireNonNull(exprType) == StringExpression.CONCAT) { - Node left = generateExpression( + PrestoExpression left = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); - Node right = generateExpression( + PrestoExpression right = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); PrestBinaryStringOperator operator = PrestBinaryStringOperator.CONCAT; - return new NewBinaryOperatorNode<>(left, right, operator); + return new PrestoBinaryOperation(left, right, operator); } throw new AssertionError(exprType); } - private Node generateBooleanExpression(int depth) { + private PrestoExpression generateBooleanExpression(int depth) { List booleanExpressions = Arrays.stream(BooleanExpression.values()) .collect(Collectors.toList()); if (!globalState.getDbmsSpecificOptions().testBetween) { @@ -292,7 +289,7 @@ private Node generateBooleanExpression(int depth) { } } - private Node getMultiValuedComparison(int depth) { + private PrestoExpression getMultiValuedComparison(int depth) { PrestoSchema.PrestoCompositeDataType type; do { @@ -304,7 +301,7 @@ private Node getMultiValuedComparison(int depth) { PrestoMultiValuedComparisonType comparisonType = PrestoMultiValuedComparisonType.getRandom(); PrestoMultiValuedComparisonOperator comparisonOperator = PrestoMultiValuedComparisonOperator .getRandomForType(type); - Node left = generateExpression(type, depth + 1); + PrestoExpression left = generateExpression(type, depth + 1); // sub-query PrestoSchema.PrestoCompositeDataType finalType = type; List columnsOfType = columns.stream().filter(c -> c.getType() == finalType) @@ -315,7 +312,7 @@ private Node getMultiValuedComparison(int depth) { return new PrestoQuantifiedComparison(left, subquery, comparisonType, comparisonOperator); } int nr = Randomly.smallNumber() + 2; - List> rightList = new ArrayList<>(); + List rightList = new ArrayList<>(); for (int i = 0; i < nr; i++) { rightList.add(generateConstant(type)); } @@ -324,53 +321,50 @@ private Node getMultiValuedComparison(int depth) { private PrestoSelect generateSubquery(List columns) { PrestoSelect select = new PrestoSelect(); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)) + List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) .collect(Collectors.toList()); select.setFetchColumns(allColumns); List tables = columns.stream().map(AbstractTableColumn::getTable) .collect(Collectors.toList()); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).distinct() + List tableList = tables.stream().map(t -> new PrestoTableReference(t)).distinct() .collect(Collectors.toList()); - List> tableNodeList = tables.stream() - .map(t -> new TableReferenceNode(t)) + List tableNodeList = tables.stream().map(t -> new PrestoTableReference(t)) .collect(Collectors.toList()); select.setFromList(tableNodeList); - TypedExpressionGenerator, PrestoSchema.PrestoColumn, PrestoSchema.PrestoCompositeDataType> typedExpressionGenerator = new PrestoTypedExpressionGenerator( + TypedExpressionGenerator typedExpressionGenerator = new PrestoTypedExpressionGenerator( globalState).setColumns(columns); - Node predicate = typedExpressionGenerator.generatePredicate(); + PrestoExpression predicate = typedExpressionGenerator.generatePredicate(); select.setWhereClause(predicate); if (Randomly.getBooleanWithSmallProbability()) { select.setOrderByClauses(typedExpressionGenerator.generateOrderBys()); } - List> joins = PrestoJoin.getJoins(tableList, globalState); + List joins = PrestoJoin.getJoins(tableList, globalState); select.setJoinList(joins); return select; } - private Node generateNumericExpression(int depth) { + private PrestoExpression generateNumericExpression(int depth) { PrestoSchema.PrestoDataType dataType = Randomly.fromList(PrestoSchema.PrestoDataType.getNumberTypes()); PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType.fromDataType(dataType); if (Randomly.getBoolean()) { BinaryOperatorNode.Operator operator = PrestoBinaryArithmeticOperator.getRandom(); - Node left = generateExpression(type, depth); - Node right = generateExpression(type, depth); - return new NewBinaryOperatorNode<>(left, right, operator); + PrestoExpression left = generateExpression(type, depth); + PrestoExpression right = generateExpression(type, depth); + return new PrestoBinaryOperation(left, right, operator); } else { BinaryOperatorNode.Operator operator = PrestoUnaryArithmeticOperator.MINUS; - Node left = generateExpression(type, depth); - return new NewUnaryPrefixOperatorNode<>(left, operator); + PrestoExpression left = generateExpression(type, depth); + return new PrestoUnaryPrefixOperation(left, operator); } } - private Node generateTemporalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateTemporalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { if (Randomly.getBooleanWithSmallProbability()) { - Node left = generateExpression(type, depth); - Node right = generateExpression(PrestoSchema.PrestoCompositeDataType + PrestoExpression left = generateExpression(type, depth); + PrestoExpression right = generateExpression(PrestoSchema.PrestoCompositeDataType .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getIntervalTypes())), depth); BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); - return new NewBinaryOperatorNode<>(left, right, operator); + return new PrestoBinaryOperation(left, right, operator); } // timestamp at time zone @@ -383,11 +377,11 @@ private Node generateTemporalExpression(PrestoSchema.PrestoCom return generateLeafNode(type); } - private Node generateIntervalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { + private PrestoExpression generateIntervalExpression(PrestoSchema.PrestoCompositeDataType type, int depth) { if (Randomly.getBooleanWithSmallProbability()) { - Node left = generateExpression(type, depth); + PrestoExpression left = generateExpression(type, depth); - Node right; + PrestoExpression right; if (Randomly.getBoolean()) { right = generateExpression(PrestoSchema.PrestoCompositeDataType .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getTemporalTypes())), depth); @@ -396,9 +390,9 @@ private Node generateIntervalExpression(PrestoSchema.PrestoCom } BinaryOperatorNode.Operator operator = PrestoBinaryTemporalOperator.getRandom(); if (Randomly.getBoolean()) { - return new NewBinaryOperatorNode<>(left, right, operator); + return new PrestoBinaryOperation(left, right, operator); } else { - return new NewBinaryOperatorNode<>(right, left, operator); + return new PrestoBinaryOperation(right, left, operator); } } return generateLeafNode(type); @@ -408,17 +402,17 @@ private Node generateIntervalExpression(PrestoSchema.PrestoCom // timestamp at time zone } - private Node getLike(int depth) { + private PrestoExpression getLike(int depth) { PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType .fromDataType(PrestoSchema.PrestoDataType.VARCHAR); - Node expression = generateExpression(type, depth + 1); - Node pattern = generateExpression(type, depth + 1); + PrestoExpression expression = generateExpression(type, depth + 1); + PrestoExpression pattern = generateExpression(type, depth + 1); if (Randomly.getBoolean()) { - return new NewBinaryOperatorNode<>(expression, pattern, PrestoLikeOperator.getRandom()); + return new PrestoBinaryOperation(expression, pattern, PrestoLikeOperator.getRandom()); } else { String randomlyString = randomly.getString(); String randomlyChar = randomly.getChar(); - Node escape = new PrestoConstant.PrestoTextConstant(randomlyChar, 1); + PrestoExpression escape = new PrestoConstant.PrestoTextConstant(randomlyChar, 1); int index = randomlyString.indexOf(randomlyChar); while (index > -1) { String wildcard = Randomly.fromOptions("%", "_"); @@ -427,73 +421,75 @@ private Node getLike(int depth) { index = randomlyString.indexOf(randomlyChar, index + 1); } PrestoConstant.PrestoTextConstant patternString = new PrestoConstant.PrestoTextConstant(randomlyString); - return new NewTernaryNode<>(expression, patternString, escape, "LIKE", "ESCAPE"); + return new PrestoTernary(expression, patternString, escape, "LIKE", "ESCAPE"); } } - private NewBinaryOperatorNode getRegex(int depth) { - Node left = generateExpression( + private PrestoBinaryOperation getRegex(int depth) { + PrestoExpression left = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); - Node right = generateExpression( + PrestoExpression right = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.VARCHAR), depth + 1); - return new NewBinaryOperatorNode<>(left, right, PrestoBinaryLogicalOperator.getRandom()); + return new PrestoBinaryOperation(left, right, PrestoBinaryLogicalOperator.getRandom()); } - private NewBinaryOperatorNode getBinaryLogical(int depth) { + private PrestoBinaryOperation getBinaryLogical(int depth) { PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType .fromDataType(PrestoSchema.PrestoDataType.BOOLEAN); - Node left = generateExpression(type, depth + 1); - Node right = generateExpression(type, depth + 1); + PrestoExpression left = generateExpression(type, depth + 1); + PrestoExpression right = generateExpression(type, depth + 1); BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode<>(left, right, operator); + return new PrestoBinaryOperation(left, right, operator); } - private Node getBetween(int depth) { + private PrestoExpression getBetween(int depth) { PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType .fromDataType(Randomly.fromList(PrestoSchema.PrestoDataType.getNumericTypes())); - Node expression = generateExpression(type, depth + 1); - Node left = generateExpression(type, depth + 1); - Node right = generateExpression(type, depth + 1); - return new NewBetweenOperatorNode<>(expression, left, right, Randomly.getBoolean()); + PrestoExpression expression = generateExpression(type, depth + 1); + PrestoExpression left = generateExpression(type, depth + 1); + PrestoExpression right = generateExpression(type, depth + 1); + return new PrestoBetweenOperation(expression, left, right, Randomly.getBoolean()); } - private Node getInOperation(int depth) { + private PrestoExpression getInOperation(int depth) { PrestoSchema.PrestoCompositeDataType type = PrestoSchema.PrestoCompositeDataType .fromDataType(PrestoSchema.PrestoDataType.getRandomWithoutNull()); - Node left = generateExpression(type, depth + 1); - List> inList = generateExpressions(type, Randomly.smallNumber() + 1, depth + 1); + PrestoExpression left = generateExpression(type, depth + 1); + List inList = generateExpressions(type, Randomly.smallNumber() + 1, depth + 1); boolean isNegated = Randomly.getBoolean(); - return new NewInOperatorNode<>(left, inList, isNegated); + return new PrestoInOperation(left, inList, isNegated); } - private Node getAndOrChain(int depth) { - Node left = generateExpression( + private PrestoExpression getAndOrChain(int depth) { + PrestoExpression left = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - Node right = generateExpression( + PrestoExpression right = generateExpression( PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth + 1); BinaryOperatorNode.Operator operator = PrestoBinaryLogicalOperator.getRandom(); - left = new NewBinaryOperatorNode<>(left, right, operator); + left = new PrestoBinaryOperation(left, right, operator); } return left; } - private Node getBinaryComparison(int depth) { + private PrestoExpression getBinaryComparison(int depth) { PrestoSchema.PrestoCompositeDataType type = getRandomType(); BinaryOperatorNode.Operator op = PrestoBinaryComparisonOperator.getRandomForType(type); - Node left = generateExpression(type, depth + 1); - Node right = generateExpression(type, depth + 1); - return new NewBinaryOperatorNode<>(left, right, op); + PrestoExpression left = generateExpression(type, depth + 1); + PrestoExpression right = generateExpression(type, depth + 1); + return new PrestoBinaryOperation(left, right, op); } - private Node generateNOT(int depth) { + private PrestoExpression generateNOT(int depth) { PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator operator = PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT; - return new PrestoUnaryPrefixOperation(operator, generateExpression( - PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth)); + return new PrestoUnaryPrefixOperation( + generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), depth), + operator); } @Override - protected Node generateColumn(PrestoSchema.PrestoCompositeDataType type) { + protected PrestoExpression generateColumn(PrestoSchema.PrestoCompositeDataType type) { List columnList = columns.stream() .filter(c -> c.getType().getPrimitiveDataType() == type.getPrimitiveDataType()) .collect(Collectors.toList()); @@ -502,7 +498,7 @@ protected Node generateColumn(PrestoSchema.PrestoCompositeData } @Override - public Node generateLeafNode(PrestoSchema.PrestoCompositeDataType type) { + public PrestoExpression generateLeafNode(PrestoSchema.PrestoCompositeDataType type) { if (Randomly.getBoolean()) { return generateConstant(type); } else { @@ -534,13 +530,13 @@ protected boolean canGenerateColumnOfType(PrestoSchema.PrestoCompositeDataType t return columns.stream().anyMatch(c -> c.getType() == type); } - public Node generateAggregate() { + public PrestoExpression generateAggregate() { PrestoAggregateFunction aggregateFunction = PrestoAggregateFunction.getRandom(); - List> argsForAggregate = generateArgsForAggregate(aggregateFunction); - return new NewFunctionNode<>(argsForAggregate, aggregateFunction); + List argsForAggregate = generateArgsForAggregate(aggregateFunction); + return new PrestoFunctionNode<>(argsForAggregate, aggregateFunction); } - public List> generateArgsForAggregate(PrestoAggregateFunction aggregateFunction) { + public List generateArgsForAggregate(PrestoAggregateFunction aggregateFunction) { PrestoSchema.PrestoCompositeDataType returnType; do { returnType = aggregateFunction.getCompositeReturnType(); @@ -548,17 +544,17 @@ public List> generateArgsForAggregate(PrestoAggregateFunc return aggregateFunction.getArgumentsForReturnType(this, this.maxDepth - 1, returnType, false); } - private Node generateAggregate(PrestoSchema.PrestoCompositeDataType type) { + private PrestoExpression generateAggregate(PrestoSchema.PrestoCompositeDataType type) { PrestoAggregateFunction aggregateFunction = Randomly .fromList(PrestoAggregateFunction.getFunctionsCompatibleWith(type)); - List> argsForAggregate = generateArgsForAggregate(type, aggregateFunction); - return new NewFunctionNode<>(argsForAggregate, aggregateFunction); + List argsForAggregate = generateArgsForAggregate(type, aggregateFunction); + return new PrestoFunctionNode<>(argsForAggregate, aggregateFunction); } - public List> generateArgsForAggregate(PrestoSchema.PrestoCompositeDataType type, + public List generateArgsForAggregate(PrestoSchema.PrestoCompositeDataType type, PrestoAggregateFunction aggregateFunction) { List returnTypes = aggregateFunction.getReturnTypes(type.getPrimitiveDataType()); - List> arguments = new ArrayList<>(); + List arguments = new ArrayList<>(); allowAggregates = false; // for (PrestoSchema.PrestoDataType argumentType : returnTypes) { arguments.add(generateExpression(PrestoSchema.PrestoCompositeDataType.fromDataType(argumentType))); @@ -568,8 +564,8 @@ public List> generateArgsForAggregate(PrestoSchema.Presto } @Override - public List> generateOrderBys() { - List> expressions = new ArrayList<>(); + public List generateOrderBys() { + List expressions = new ArrayList<>(); int nr = Randomly.smallNumber() + 1; ArrayList prestoColumns = new ArrayList<>(columns); prestoColumns.removeIf(c -> !c.isOrderable()); @@ -582,15 +578,14 @@ public List> generateOrderBys() { return expressions; } - public Node generateHavingClause() { + public PrestoExpression generateHavingClause() { allowAggregates = true; - Node expr = generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); + PrestoExpression expr = generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); allowAggregates = false; return expr; } - public Node generateExpressionWithColumns(List columns, - int remainingDepth) { + public PrestoExpression generateExpressionWithColumns(List columns, int remainingDepth) { if (columns.isEmpty() || remainingDepth <= 2 && Randomly.getBooleanWithRatherLowProbability()) { return generateConstant(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); } @@ -613,7 +608,7 @@ public Node generateExpressionWithColumns(List(generateExpression(column.getType(), remainingDepth - 1), + return new PrestoBinaryOperation(generateExpression(column.getType(), remainingDepth - 1), generateExpression(column.getType(), remainingDepth - 1), op); } diff --git a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java index bdd8cf2a3..a8afcc578 100644 --- a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java +++ b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoErrors; @@ -40,7 +39,7 @@ private SQLQueryAdapter generate() { @Override protected void updateValue(PrestoColumn column) { - Node expr; + PrestoExpression expr; if (Randomly.getBooleanWithSmallProbability()) { expr = gen.generateExpression(column.getType()); PrestoErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java index da09b5a01..606183828 100644 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -10,10 +10,6 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewPostfixTextNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.oracle.NoRECBase; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; @@ -28,9 +24,12 @@ import sqlancer.presto.PrestoSchema.PrestoTables; import sqlancer.presto.PrestoToStringVisitor; import sqlancer.presto.ast.PrestoCastFunction; +import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoExpression; import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoPostfixText; import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; public class PrestoNoRECOracle extends NoRECBase implements TestOracle { @@ -50,12 +49,12 @@ public void check() throws SQLException { List tables = randomTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = PrestoJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, state); PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(state).setColumns(columns); - Node randomWhereCondition = gen.generatePredicate(); + PrestoExpression randomWhereCondition = gen.generatePredicate(); int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); @@ -68,14 +67,14 @@ public void check() throws SQLException { } } - private int getSecondQuery(List> tableList, Node randomWhereCondition, - List> joins) throws SQLException { + private int getSecondQuery(List tableList, PrestoExpression randomWhereCondition, + List joins) throws SQLException { PrestoSelect select = new PrestoSelect(); - Node asText = new NewPostfixTextNode<>( + PrestoExpression asText = new PrestoPostfixText( new PrestoCastFunction( - new NewPostfixTextNode<>(randomWhereCondition, + new PrestoPostfixText(randomWhereCondition, " IS NOT NULL AND " + PrestoToStringVisitor.asString(randomWhereCondition)), new PrestoCompositeDataType(PrestoDataType.INT, 8, 0)), "as count"); @@ -104,12 +103,11 @@ private int getSecondQuery(List> tableList, Node> tableList, - List columns, Node randomWhereCondition, - List> joins) { + private int getFirstQueryCount(SQLConnection con, List tableList, List columns, + PrestoExpression randomWhereCondition, List joins) { PrestoSelect select = new PrestoSelect(); - List> allColumns = columns.stream() - .map((c) -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); select.setFetchColumns(allColumns); select.setFromList(tableList); select.setWhereClause(randomWhereCondition); diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java index 0f10a22d4..7ab23286a 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningAggregateTester.java @@ -8,11 +8,6 @@ import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.ast.newast.NewAliasNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; @@ -22,8 +17,10 @@ import sqlancer.presto.PrestoSchema.PrestoDataType; import sqlancer.presto.PrestoToStringVisitor; import sqlancer.presto.ast.PrestoAggregateFunction; +import sqlancer.presto.ast.PrestoAlias; import sqlancer.presto.ast.PrestoCastFunction; import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoFunctionNode; import sqlancer.presto.ast.PrestoSelect; import sqlancer.presto.ast.PrestoUnaryPostfixOperation; import sqlancer.presto.ast.PrestoUnaryPrefixOperation; @@ -49,8 +46,8 @@ public void check() throws SQLException { PrestoAggregateFunction aggregateFunction = Randomly.fromOptions(PrestoAggregateFunction.MAX, PrestoAggregateFunction.MIN, PrestoAggregateFunction.SUM, PrestoAggregateFunction.COUNT, PrestoAggregateFunction.AVG/* , PrestoAggregateFunction.STDDEV_POP */); - List> aggregateArgs = gen.generateArgsForAggregate(aggregateFunction); - NewFunctionNode aggregate = new NewFunctionNode<>(aggregateArgs, + List aggregateArgs = gen.generateArgsForAggregate(aggregateFunction); + PrestoFunctionNode aggregate = new PrestoFunctionNode<>(aggregateArgs, aggregateFunction); select.setFetchColumns(List.of(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { @@ -84,14 +81,14 @@ public void check() throws SQLException { } private String createMetamorphicUnionQuery(PrestoSelect select, - NewFunctionNode aggregate, List> from) { + PrestoFunctionNode aggregate, List from) { String metamorphicQuery; - Node whereClause = gen.generatePredicate(); - Node negatedClause = new NewUnaryPrefixOperatorNode<>(whereClause, + PrestoExpression whereClause = gen.generatePredicate(); + PrestoExpression negatedClause = new PrestoUnaryPrefixOperation(whereClause, PrestoUnaryPrefixOperation.PrestoUnaryPrefixOperator.NOT); - Node notNullClause = new NewUnaryPostfixOperatorNode<>(whereClause, + PrestoExpression notNullClause = new PrestoUnaryPostfixOperation(whereClause, PrestoUnaryPostfixOperation.PrestoUnaryPostfixOperator.IS_NULL); - List> mappedAggregate = mapped(aggregate); + List mappedAggregate = mapped(aggregate); PrestoSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinList()); PrestoSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinList()); PrestoSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinList()); @@ -151,7 +148,7 @@ private String getAggregateResultType(String queryString) { } } - private List> mapped(NewFunctionNode aggregate) { + private List mapped(PrestoFunctionNode aggregate) { PrestoCastFunction count; switch (aggregate.getFunc()) { case COUNT: @@ -160,9 +157,9 @@ private List> mapped(NewFunctionNode sum = new NewFunctionNode<>(aggregate.getArgs(), + PrestoFunctionNode sum = new PrestoFunctionNode<>(aggregate.getArgs(), PrestoAggregateFunction.SUM); - count = new PrestoCastFunction(new NewFunctionNode<>(aggregate.getArgs(), PrestoAggregateFunction.COUNT), + count = new PrestoCastFunction(new PrestoFunctionNode<>(aggregate.getArgs(), PrestoAggregateFunction.COUNT), new PrestoCompositeDataType(PrestoDataType.FLOAT, 8, 0)); return aliasArgs(Arrays.asList(sum, count)); default: @@ -170,16 +167,16 @@ private List> mapped(NewFunctionNode> aliasArgs(List> originalAggregateArgs) { - List> args = new ArrayList<>(); + private List aliasArgs(List originalAggregateArgs) { + List args = new ArrayList<>(); int i = 0; - for (Node expr : originalAggregateArgs) { - args.add(new NewAliasNode<>(expr, "agg" + i++)); + for (PrestoExpression expr : originalAggregateArgs) { + args.add(new PrestoAlias(expr, "agg" + i++)); } return args; } - private String getOuterAggregateFunction(NewFunctionNode aggregate) { + private String getOuterAggregateFunction(PrestoFunctionNode aggregate) { switch (aggregate.getFunc()) { case AVG: return "SUM(CAST(agg0 AS DOUBLE))/CAST(SUM(agg1) AS DOUBLE)"; @@ -190,8 +187,8 @@ private String getOuterAggregateFunction(NewFunctionNode> aggregates, List> from, - Node whereClause, List> joinList) { + private PrestoSelect getSelect(List aggregates, List from, + PrestoExpression whereClause, List joinList) { PrestoSelect leftSelect = new PrestoSelect(); leftSelect.setFetchColumns(aggregates); leftSelect.setFromList(from); diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java index 5ee56e1c8..9ac6ef9f4 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java @@ -6,9 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -18,13 +15,14 @@ import sqlancer.presto.PrestoSchema.PrestoColumn; import sqlancer.presto.PrestoSchema.PrestoTable; import sqlancer.presto.PrestoSchema.PrestoTables; +import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoExpression; import sqlancer.presto.ast.PrestoJoin; import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.ast.PrestoTableReference; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, PrestoGlobalState> +public class PrestoQueryPartitioningBase extends TernaryLogicPartitioningOracleBase implements TestOracle { PrestoSchema s; @@ -63,27 +61,27 @@ public void check() throws SQLException { select = new PrestoSelect(); select.setFetchColumns(generateFetchColumns()); List tables = targetTables.getTables(); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); - List> joins = PrestoJoin.getJoins(tableList, state); + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, state); select.setJoinList(new ArrayList<>(joins)); select.setFromList(new ArrayList<>(tableList)); select.setWhereClause(null); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new PrestoColumn("*", null, false, false))); + columns.add(new PrestoColumnReference(new PrestoColumn("*", null, false, false))); } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); } return columns; } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java index 5d5e2343f..6c3f4bf71 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningGroupByTester.java @@ -7,12 +7,10 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; -import sqlancer.presto.PrestoSchema.PrestoColumn; import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoExpression; public class PrestoQueryPartitioningGroupByTester extends PrestoQueryPartitioningBase { @@ -45,9 +43,9 @@ public void check() throws SQLException { } @Override - List> generateFetchColumns() { - return Randomly.nonEmptySubset(targetTables.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)).collect(Collectors.toList()); + List generateFetchColumns() { + return Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); } } diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java index ad11806bf..b53bfc07f 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningHavingTester.java @@ -7,7 +7,6 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.oracle.TestOracle; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; @@ -52,12 +51,12 @@ public void check() throws SQLException { } @Override - protected Node generatePredicate() { + protected PrestoExpression generatePredicate() { return gen.generateHavingClause(); } @Override - List> generateFetchColumns() { + List generateFetchColumns() { return Collections.singletonList(gen.generateHavingClause()); } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java new file mode 100644 index 000000000..1f648ed35 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class YCQLBetweenOperator extends NewBetweenOperatorNode implements YCQLExpression { + public YCQLBetweenOperator(YCQLExpression left, YCQLExpression middle, YCQLExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java new file mode 100644 index 000000000..369884e77 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBBinaryOperation extends NewBinaryOperatorNode implements QuestDBExpression { + public QuestDBBinaryOperation(QuestDBExpression left, QuestDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java new file mode 100644 index 000000000..33ba8b619 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java @@ -0,0 +1,13 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.questdb.QuestDBSchema; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBColumnReference extends ColumnReferenceNode + implements QuestDBExpression { + public QuestDBColumnReference(QuestDBSchema.QuestDBColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java new file mode 100644 index 000000000..01c880509 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; +import sqlancer.duckdb.ast.DuckDBExpression; + +public class DuckDBFunction extends NewFunctionNode implements DuckDBExpression { + public DuckDBFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java new file mode 100644 index 000000000..b050276ea --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBInOperation extends NewInOperatorNode implements QuestDBExpression { + public QuestDBInOperation(QuestDBExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java new file mode 100644 index 000000000..575725a07 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java @@ -0,0 +1,10 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.duckdb.ast.DuckDBExpression; + +public class DuckDBOrderingTerm extends NewOrderingTerm implements DuckDBExpression { + public DuckDBOrderingTerm(DuckDBExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java new file mode 100644 index 000000000..1a5f5e638 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.questdb.QuestDBSchema; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBTableReference extends TableReferenceNode + implements QuestDBExpression { + public QuestDBTableReference(QuestDBSchema.QuestDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java new file mode 100644 index 000000000..a6f63860a --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPostfixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java new file mode 100644 index 000000000..e9546ba8b --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.yugabyte.ycql.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; +import sqlancer.questdb.ast.QuestDBExpression; + +public class QuestDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPrefixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} From b3f9c16595189c81f265ac4e601379ee73556393 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 13:31:58 +0800 Subject: [PATCH 476/856] [QuestDB] Update expressions to match new AST --- .../questdb/QuestDBToStringVisitor.java | 5 +-- .../questdb/ast/QuestDBBinaryOperation.java | 10 +++++ .../questdb/ast/QuestDBColumnReference.java | 12 ++++++ src/sqlancer/questdb/ast/QuestDBConstant.java | 14 +++---- .../questdb/ast/QuestDBInOperation.java | 11 +++++ src/sqlancer/questdb/ast/QuestDBSelect.java | 3 +- .../questdb/ast/QuestDBTableReference.java | 11 +++++ .../ast/QuestDBUnaryPostfixOperation.java | 11 +++++ .../ast/QuestDBUnaryPrefixOperation.java | 11 +++++ .../gen/QuestDBExpressionGenerator.java | 41 +++++++++---------- .../test/QuestDBQueryPartitioningBase.java | 22 +++++----- 11 files changed, 105 insertions(+), 46 deletions(-) create mode 100644 src/sqlancer/questdb/ast/QuestDBBinaryOperation.java create mode 100644 src/sqlancer/questdb/ast/QuestDBColumnReference.java create mode 100644 src/sqlancer/questdb/ast/QuestDBInOperation.java create mode 100644 src/sqlancer/questdb/ast/QuestDBTableReference.java create mode 100644 src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java create mode 100644 src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java diff --git a/src/sqlancer/questdb/QuestDBToStringVisitor.java b/src/sqlancer/questdb/QuestDBToStringVisitor.java index 7b2c889d2..9363b30b1 100644 --- a/src/sqlancer/questdb/QuestDBToStringVisitor.java +++ b/src/sqlancer/questdb/QuestDBToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.questdb; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.questdb.ast.QuestDBConstant; import sqlancer.questdb.ast.QuestDBExpression; import sqlancer.questdb.ast.QuestDBSelect; @@ -9,7 +8,7 @@ public class QuestDBToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(QuestDBExpression expr) { if (expr instanceof QuestDBConstant) { visit((QuestDBConstant) expr); } else if (expr instanceof QuestDBSelect) { @@ -59,7 +58,7 @@ private void visit(QuestDBSelect select) { } } - public static String asString(Node expr) { + public static String asString(QuestDBExpression expr) { QuestDBToStringVisitor visitor = new QuestDBToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/questdb/ast/QuestDBBinaryOperation.java b/src/sqlancer/questdb/ast/QuestDBBinaryOperation.java new file mode 100644 index 000000000..1b69b2c31 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBBinaryOperation.java @@ -0,0 +1,10 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class QuestDBBinaryOperation extends NewBinaryOperatorNode implements QuestDBExpression { + public QuestDBBinaryOperation(QuestDBExpression left, QuestDBExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBColumnReference.java b/src/sqlancer/questdb/ast/QuestDBColumnReference.java new file mode 100644 index 000000000..d627538a0 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.questdb.QuestDBSchema; + +public class QuestDBColumnReference extends ColumnReferenceNode + implements QuestDBExpression { + public QuestDBColumnReference(QuestDBSchema.QuestDBColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/questdb/ast/QuestDBConstant.java b/src/sqlancer/questdb/ast/QuestDBConstant.java index f7e5ab0f8..f2e65fbc0 100644 --- a/src/sqlancer/questdb/ast/QuestDBConstant.java +++ b/src/sqlancer/questdb/ast/QuestDBConstant.java @@ -1,8 +1,6 @@ package sqlancer.questdb.ast; -import sqlancer.common.ast.newast.Node; - -public class QuestDBConstant implements Node { +public class QuestDBConstant implements QuestDBExpression { private QuestDBConstant() { } @@ -67,7 +65,7 @@ public String getValue() { } } - public static Node createIntConstant(long val) { + public static QuestDBExpression createIntConstant(long val) { return new QuestDBIntConstant(val); } @@ -95,19 +93,19 @@ public String toString() { } - public static Node createBooleanConstant(boolean val) { + public static QuestDBExpression createBooleanConstant(boolean val) { return new QuestDBBooleanConstant(val); } - public static Node createNullConstant() { + public static QuestDBExpression createNullConstant() { return new QuestDBNullConstant(); } - public static Node createFloatConstant(double val) { + public static QuestDBExpression createFloatConstant(double val) { return new QuestDBDoubleConstant(val); } - public static Node createSymbolConstant(String val) { + public static QuestDBExpression createSymbolConstant(String val) { return new QuestDBSymbolConstant(val); } } diff --git a/src/sqlancer/questdb/ast/QuestDBInOperation.java b/src/sqlancer/questdb/ast/QuestDBInOperation.java new file mode 100644 index 000000000..ee4ba2b59 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBInOperation.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class QuestDBInOperation extends NewInOperatorNode implements QuestDBExpression { + public QuestDBInOperation(QuestDBExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBSelect.java b/src/sqlancer/questdb/ast/QuestDBSelect.java index 023384487..0684b4082 100644 --- a/src/sqlancer/questdb/ast/QuestDBSelect.java +++ b/src/sqlancer/questdb/ast/QuestDBSelect.java @@ -1,9 +1,8 @@ package sqlancer.questdb.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class QuestDBSelect extends SelectBase> implements Node { +public class QuestDBSelect extends SelectBase implements QuestDBExpression { private boolean isDistinct; public void setDistinct(boolean distinct) { diff --git a/src/sqlancer/questdb/ast/QuestDBTableReference.java b/src/sqlancer/questdb/ast/QuestDBTableReference.java new file mode 100644 index 000000000..8e4d387da --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.questdb.QuestDBSchema; + +public class QuestDBTableReference extends TableReferenceNode + implements QuestDBExpression { + public QuestDBTableReference(QuestDBSchema.QuestDBTable table) { + super(table); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java b/src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java new file mode 100644 index 000000000..0301ba60a --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBUnaryPostfixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class QuestDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPostfixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java b/src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java new file mode 100644 index 000000000..8488a0de8 --- /dev/null +++ b/src/sqlancer/questdb/ast/QuestDBUnaryPrefixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.questdb.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class QuestDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements QuestDBExpression { + public QuestDBUnaryPrefixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java index 73f57c055..cbb19e705 100644 --- a/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBExpressionGenerator.java @@ -7,20 +7,19 @@ import sqlancer.Randomly; import sqlancer.Randomly.StringGenerationStrategy; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBDataType; +import sqlancer.questdb.ast.QuestDBBinaryOperation; +import sqlancer.questdb.ast.QuestDBColumnReference; import sqlancer.questdb.ast.QuestDBConstant; import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.questdb.ast.QuestDBInOperation; +import sqlancer.questdb.ast.QuestDBUnaryPostfixOperation; +import sqlancer.questdb.ast.QuestDBUnaryPrefixOperation; -public class QuestDBExpressionGenerator extends UntypedExpressionGenerator, QuestDBColumn> { +public class QuestDBExpressionGenerator extends UntypedExpressionGenerator { private final QuestDBGlobalState globalState; @@ -33,17 +32,17 @@ private enum Expression { } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, QuestDBUnaryPrefixOperator.NOT); + public QuestDBExpression negatePredicate(QuestDBExpression predicate) { + return new QuestDBUnaryPrefixOperation(predicate, QuestDBUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, QuestDBUnaryPostfixOperator.IS_NULL); + public QuestDBExpression isNull(QuestDBExpression expr) { + return new QuestDBUnaryPostfixOperation(expr, QuestDBUnaryPostfixOperator.IS_NULL); } @Override - public Node generateConstant() { + public QuestDBExpression generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { return QuestDBConstant.createNullConstant(); } @@ -68,7 +67,7 @@ public Node generateConstant() { } @Override - protected Node generateExpression(int depth) { + protected QuestDBExpression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); } @@ -78,22 +77,22 @@ protected Node generateExpression(int depth) { switch (expr) { case UNARY_PREFIX: - return new NewUnaryPrefixOperatorNode<>(generateExpression(depth + 1), + return new QuestDBUnaryPrefixOperation(generateExpression(depth + 1), QuestDBUnaryPrefixOperator.getRandom()); case UNARY_POSTFIX: - return new NewUnaryPostfixOperatorNode<>(generateExpression(depth + 1), + return new QuestDBUnaryPostfixOperation(generateExpression(depth + 1), QuestDBUnaryPostfixOperator.getRandom()); case BINARY_COMPARISON: - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + return new QuestDBBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), QuestDBBinaryComparisonOperator.getRandom()); case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + return new QuestDBBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), QuestDBBinaryArithmeticOperator.getRandom()); case BINARY_LOGICAL: - return new NewBinaryOperatorNode<>(generateExpression(depth + 1), generateExpression(depth + 1), + return new QuestDBBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), QuestDBBinaryLogicalOperator.getRandom()); case IN: - return new NewInOperatorNode<>(generateExpression(depth + 1), + return new QuestDBInOperation(generateExpression(depth + 1), generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); default: throw new AssertionError("Expression generation failed, depth=" + depth); @@ -101,9 +100,9 @@ protected Node generateExpression(int depth) { } @Override - protected Node generateColumn() { + protected QuestDBExpression generateColumn() { QuestDBColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode<>(column); + return new QuestDBColumnReference(column); } public enum QuestDBUnaryPostfixOperator implements Operator { diff --git a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java index 068abbd38..0716e9ccc 100644 --- a/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java +++ b/src/sqlancer/questdb/test/QuestDBQueryPartitioningBase.java @@ -6,9 +6,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -17,12 +14,14 @@ import sqlancer.questdb.QuestDBSchema; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBTable; +import sqlancer.questdb.ast.QuestDBColumnReference; import sqlancer.questdb.ast.QuestDBExpression; import sqlancer.questdb.ast.QuestDBSelect; +import sqlancer.questdb.ast.QuestDBTableReference; import sqlancer.questdb.gen.QuestDBExpressionGenerator; public class QuestDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, QuestDBGlobalState> + extends TernaryLogicPartitioningOracleBase implements TestOracle { QuestDBSchema s; @@ -35,20 +34,19 @@ protected QuestDBQueryPartitioningBase(QuestDBGlobalState state) { QuestDBErrors.addExpressionErrors(errors); } - List> generateFetchColumns() { - List> columns = new ArrayList<>(); + List generateFetchColumns() { + List columns = new ArrayList<>(); if (Randomly.getBoolean()) { - columns.add(new ColumnReferenceNode<>(new QuestDBColumn("*", null, false))); + columns.add(new QuestDBColumnReference(new QuestDBColumn("*", null, false))); } else { - columns = Randomly.nonEmptySubset(targetTable.getColumns()).stream() - .map(c -> new ColumnReferenceNode(c)) + columns = Randomly.nonEmptySubset(targetTable.getColumns()).stream().map(c -> new QuestDBColumnReference(c)) .collect(Collectors.toList()); } return columns; } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } @@ -64,8 +62,8 @@ public void check() throws SQLException { select.setFetchColumns(generateFetchColumns()); List tables = new ArrayList<>(); tables.add(targetTable); - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)).collect(Collectors.toList()); + List tableList = tables.stream().map(t -> new QuestDBTableReference(t)) + .collect(Collectors.toList()); // Ignore JOINs for now select.setFromList(new ArrayList<>(tableList)); select.setWhereClause(null); From 0ddd0ee2c5aa0169c5dc5b7ddd123b6066fc2155 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 13:53:25 +0800 Subject: [PATCH 477/856] [YCQL] Update expressions to match new AST --- .../yugabyte/ycql/YCQLToStringVisitor.java | 5 +- .../ycql/ast/YCQLBetweenOperation.java | 5 +- .../ycql/ast/YCQLBinaryOperation.java | 5 +- .../ycql/ast/YCQLColumnReference.java | 9 +-- .../yugabyte/ycql/ast/YCQLConstant.java | 18 ++--- .../yugabyte/ycql/ast/YCQLFunction.java | 5 +- .../yugabyte/ycql/ast/YCQLInOperation.java | 5 +- .../yugabyte/ycql/ast/YCQLOrderingTerm.java | 5 +- .../yugabyte/ycql/ast/YCQLSelect.java | 3 +- .../yugabyte/ycql/ast/YCQLTableReference.java | 9 +-- .../ycql/ast/YCQLUnaryPostfixOperation.java | 6 +- .../ycql/ast/YCQLUnaryPrefixOperation.java | 6 +- .../ycql/gen/YCQLExpressionGenerator.java | 74 +++++++++---------- .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 3 +- .../ycql/gen/YCQLRandomQuerySynthesizer.java | 10 +-- .../ycql/gen/YCQLUpdateGenerator.java | 3 +- 16 files changed, 74 insertions(+), 97 deletions(-) diff --git a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java index 59adf3e6b..c9f357fe9 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java +++ b/src/sqlancer/yugabyte/ycql/YCQLToStringVisitor.java @@ -1,7 +1,6 @@ package sqlancer.yugabyte.ycql; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; import sqlancer.yugabyte.ycql.ast.YCQLSelect; @@ -9,7 +8,7 @@ public class YCQLToStringVisitor extends NewToStringVisitor { @Override - public void visitSpecific(Node expr) { + public void visitSpecific(YCQLExpression expr) { if (expr instanceof YCQLConstant) { visit((YCQLConstant) expr); } else if (expr instanceof YCQLSelect) { @@ -55,7 +54,7 @@ private void visit(YCQLSelect select) { } } - public static String asString(Node expr) { + public static String asString(YCQLExpression expr) { YCQLToStringVisitor visitor = new YCQLToStringVisitor(); visitor.visit(expr); return visitor.get(); diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java index 1f648ed35..880007cd8 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBetweenOperation.java @@ -2,9 +2,8 @@ import sqlancer.common.ast.newast.NewBetweenOperatorNode; -public class YCQLBetweenOperator extends NewBetweenOperatorNode implements YCQLExpression { - public YCQLBetweenOperator(YCQLExpression left, YCQLExpression middle, YCQLExpression right, - boolean isTrue) { +public class YCQLBetweenOperation extends NewBetweenOperatorNode implements YCQLExpression { + public YCQLBetweenOperation(YCQLExpression left, YCQLExpression middle, YCQLExpression right, boolean isTrue) { super(left, middle, right, isTrue); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java index 369884e77..fac2af681 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLBinaryOperation.java @@ -2,10 +2,9 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.questdb.ast.QuestDBExpression; -public class QuestDBBinaryOperation extends NewBinaryOperatorNode implements QuestDBExpression { - public QuestDBBinaryOperation(QuestDBExpression left, QuestDBExpression right, Operator op) { +public class YCQLBinaryOperation extends NewBinaryOperatorNode implements YCQLExpression { + public YCQLBinaryOperation(YCQLExpression left, YCQLExpression right, Operator op) { super(left, right, op); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java index 33ba8b619..9c81e70a5 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLColumnReference.java @@ -1,12 +1,11 @@ package sqlancer.yugabyte.ycql.ast; import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.questdb.QuestDBSchema; -import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.yugabyte.ycql.YCQLSchema; -public class QuestDBColumnReference extends ColumnReferenceNode - implements QuestDBExpression { - public QuestDBColumnReference(QuestDBSchema.QuestDBColumn column) { +public class YCQLColumnReference extends ColumnReferenceNode + implements YCQLExpression { + public YCQLColumnReference(YCQLSchema.YCQLColumn column) { super(column); } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java index dd85b4b66..08f04bcfb 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLConstant.java @@ -3,9 +3,7 @@ import java.sql.Timestamp; import java.text.SimpleDateFormat; -import sqlancer.common.ast.newast.Node; - -public class YCQLConstant implements Node { +public class YCQLConstant implements YCQLExpression { private YCQLConstant() { } @@ -142,31 +140,31 @@ public String toString() { } - public static Node createStringConstant(String text) { + public static YCQLExpression createStringConstant(String text) { return new YCQLTextConstant(text); } - public static Node createFloatConstant(double val) { + public static YCQLExpression createFloatConstant(double val) { return new YCQLDoubleConstant(val); } - public static Node createIntConstant(long val) { + public static YCQLExpression createIntConstant(long val) { return new YCQLIntConstant(val); } - public static Node createNullConstant() { + public static YCQLExpression createNullConstant() { return new YCQLNullConstant(); } - public static Node createBooleanConstant(boolean val) { + public static YCQLExpression createBooleanConstant(boolean val) { return new YCQLBooleanConstant(val); } - public static Node createDateConstant(long integer) { + public static YCQLExpression createDateConstant(long integer) { return new YCQLDateConstant(integer); } - public static Node createTimestampConstant(long integer) { + public static YCQLExpression createTimestampConstant(long integer) { return new YCQLTimestampConstant(integer); } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java index 01c880509..483fab4c4 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLFunction.java @@ -3,10 +3,9 @@ import java.util.List; import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBFunction extends NewFunctionNode implements DuckDBExpression { - public DuckDBFunction(List args, F func) { +public class YCQLFunction extends NewFunctionNode implements YCQLExpression { + public YCQLFunction(List args, F func) { super(args, func); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java index b050276ea..49d43be11 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLInOperation.java @@ -3,10 +3,9 @@ import java.util.List; import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.questdb.ast.QuestDBExpression; -public class QuestDBInOperation extends NewInOperatorNode implements QuestDBExpression { - public QuestDBInOperation(QuestDBExpression left, List right, boolean isNegated) { +public class YCQLInOperation extends NewInOperatorNode implements YCQLExpression { + public YCQLInOperation(YCQLExpression left, List right, boolean isNegated) { super(left, right, isNegated); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java index 575725a07..5f647e721 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLOrderingTerm.java @@ -1,10 +1,9 @@ package sqlancer.yugabyte.ycql.ast; import sqlancer.common.ast.newast.NewOrderingTerm; -import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBOrderingTerm extends NewOrderingTerm implements DuckDBExpression { - public DuckDBOrderingTerm(DuckDBExpression expr, Ordering ordering) { +public class YCQLOrderingTerm extends NewOrderingTerm implements YCQLExpression { + public YCQLOrderingTerm(YCQLExpression expr, Ordering ordering) { super(expr, ordering); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java index 646182875..25257057e 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLSelect.java @@ -1,9 +1,8 @@ package sqlancer.yugabyte.ycql.ast; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.Node; -public class YCQLSelect extends SelectBase> implements Node { +public class YCQLSelect extends SelectBase implements YCQLExpression { private boolean isDistinct; diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java index 1a5f5e638..70c01b820 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLTableReference.java @@ -1,12 +1,11 @@ package sqlancer.yugabyte.ycql.ast; import sqlancer.common.ast.newast.TableReferenceNode; -import sqlancer.questdb.QuestDBSchema; -import sqlancer.questdb.ast.QuestDBExpression; +import sqlancer.yugabyte.ycql.YCQLSchema; -public class QuestDBTableReference extends TableReferenceNode - implements QuestDBExpression { - public QuestDBTableReference(QuestDBSchema.QuestDBTable table) { +public class YCQLTableReference extends TableReferenceNode + implements YCQLExpression { + public YCQLTableReference(YCQLSchema.YCQLTable table) { super(table); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java index a6f63860a..b9165211a 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPostfixOperation.java @@ -2,11 +2,9 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.questdb.ast.QuestDBExpression; -public class QuestDBUnaryPostfixOperation extends NewUnaryPostfixOperatorNode - implements QuestDBExpression { - public QuestDBUnaryPostfixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator op) { +public class YCQLUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements YCQLExpression { + public YCQLUnaryPostfixOperation(YCQLExpression expr, BinaryOperatorNode.Operator op) { super(expr, op); } } diff --git a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java index e9546ba8b..bd3b554ee 100644 --- a/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java +++ b/src/sqlancer/yugabyte/ycql/ast/YCQLUnaryPrefixOperation.java @@ -2,11 +2,9 @@ import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.questdb.ast.QuestDBExpression; -public class QuestDBUnaryPrefixOperation extends NewUnaryPrefixOperatorNode - implements QuestDBExpression { - public QuestDBUnaryPrefixOperation(QuestDBExpression expr, BinaryOperatorNode.Operator operator) { +public class YCQLUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements YCQLExpression { + public YCQLUnaryPrefixOperation(YCQLExpression expr, BinaryOperatorNode.Operator operator) { super(expr, operator); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java index 7a0a7ed1b..0b0e4dee5 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLExpressionGenerator.java @@ -9,24 +9,23 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBetweenOperatorNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; -import sqlancer.common.ast.newast.NewInOperatorNode; -import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; -import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLDataType; +import sqlancer.yugabyte.ycql.ast.YCQLBetweenOperation; +import sqlancer.yugabyte.ycql.ast.YCQLBinaryOperation; +import sqlancer.yugabyte.ycql.ast.YCQLColumnReference; import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; +import sqlancer.yugabyte.ycql.ast.YCQLFunction; +import sqlancer.yugabyte.ycql.ast.YCQLInOperation; +import sqlancer.yugabyte.ycql.ast.YCQLOrderingTerm; +import sqlancer.yugabyte.ycql.ast.YCQLUnaryPostfixOperation; +import sqlancer.yugabyte.ycql.ast.YCQLUnaryPrefixOperation; -public final class YCQLExpressionGenerator extends UntypedExpressionGenerator, YCQLColumn> { +public final class YCQLExpressionGenerator extends UntypedExpressionGenerator { private final YCQLGlobalState globalState; @@ -39,37 +38,35 @@ private enum Expression { } @Override - protected Node generateExpression(int depth) { + protected YCQLExpression generateExpression(int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { return generateLeafNode(); } if (allowAggregates && Randomly.getBoolean()) { YCQLAggregateFunction aggregate = YCQLAggregateFunction.getRandom(); allowAggregates = false; - return new NewFunctionNode<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); + return new YCQLFunction<>(generateExpressions(depth + 1, aggregate.getNrArgs()), aggregate); } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); Expression expr = Randomly.fromList(possibleOptions); switch (expr) { case BINARY_COMPARISON: Operator op = YCQLBinaryComparisonOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new YCQLBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_LOGICAL: op = YCQLBinaryLogicalOperator.getRandom(); - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), op); + return new YCQLBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); case BINARY_ARITHMETIC: - return new NewBinaryOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), YCQLBinaryArithmeticOperator.getRandom()); + return new YCQLBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + YCQLBinaryArithmeticOperator.getRandom()); case FUNC: DBFunction func = DBFunction.getRandom(); - return new NewFunctionNode(generateExpressions(func.getNrArgs()), func); + return new YCQLFunction(generateExpressions(func.getNrArgs()), func); case BETWEEN: - return new NewBetweenOperatorNode(generateExpression(depth + 1), - generateExpression(depth + 1), generateExpression(depth + 1), Randomly.getBoolean()); + return new YCQLBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); case IN: - return new NewInOperatorNode(generateExpression(depth + 1), + return new YCQLInOperation(generateExpression(depth + 1), generateExpressions(depth + 1, Randomly.smallNumber() + 1), Randomly.getBoolean()); default: throw new AssertionError(expr); @@ -77,13 +74,13 @@ protected Node generateExpression(int depth) { } @Override - protected Node generateColumn() { + protected YCQLExpression generateColumn() { YCQLColumn column = Randomly.fromList(columns); - return new ColumnReferenceNode(column); + return new YCQLColumnReference(column); } @Override - public Node generateConstant() { + public YCQLExpression generateConstant() { if (Randomly.getBooleanWithSmallProbability()) { if (bug14330) { throw new IgnoreMeException(); @@ -111,12 +108,12 @@ public Node generateConstant() { } @Override - public List> generateOrderBys() { - List> expr = super.generateOrderBys(); - List> newExpr = new ArrayList<>(expr.size()); - for (Node curExpr : expr) { + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (YCQLExpression curExpr : expr) { if (Randomly.getBoolean()) { - curExpr = new NewOrderingTerm<>(curExpr, Ordering.getRandom()); + curExpr = new YCQLOrderingTerm(curExpr, Ordering.getRandom()); } newExpr.add(curExpr); } @@ -280,25 +277,24 @@ public String getTextRepresentation() { } - public NewFunctionNode generateArgsForAggregate( - YCQLAggregateFunction aggregateFunction) { - return new NewFunctionNode( - generateExpressions(aggregateFunction.getNrArgs()), aggregateFunction); + public YCQLFunction generateArgsForAggregate(YCQLAggregateFunction aggregateFunction) { + return new YCQLFunction(generateExpressions(aggregateFunction.getNrArgs()), + aggregateFunction); } - public Node generateAggregate() { + public YCQLExpression generateAggregate() { YCQLAggregateFunction aggrFunc = YCQLAggregateFunction.getRandom(); return generateArgsForAggregate(aggrFunc); } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, YCQLUnaryPrefixOperator.NOT); + public YCQLExpression negatePredicate(YCQLExpression predicate) { + return new YCQLUnaryPrefixOperation(predicate, YCQLUnaryPrefixOperator.NOT); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, YCQLUnaryPostfixOperator.IS_NULL); + public YCQLExpression isNull(YCQLExpression expr) { + return new YCQLUnaryPostfixOperation(expr, YCQLUnaryPostfixOperator.IS_NULL); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java index 2dc565196..ab03316cc 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; @@ -41,7 +40,7 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { sb.append(")"); if (Randomly.getBoolean()) { sb.append(" WHERE "); - Node expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) + YCQLExpression expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(); sb.append(YCQLToStringVisitor.asString(expr)); } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java index 05f9ec2dd..f7692ebb0 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLRandomQuerySynthesizer.java @@ -6,14 +6,13 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTables; import sqlancer.yugabyte.ycql.ast.YCQLConstant; import sqlancer.yugabyte.ycql.ast.YCQLExpression; import sqlancer.yugabyte.ycql.ast.YCQLSelect; +import sqlancer.yugabyte.ycql.ast.YCQLTableReference; public final class YCQLRandomQuerySynthesizer { @@ -24,15 +23,14 @@ public static YCQLSelect generateSelect(YCQLGlobalState globalState, int nrColum YCQLTables targetTables = globalState.getSchema().getRandomTableNonEmptyTables(); YCQLExpressionGenerator gen = new YCQLExpressionGenerator(globalState).setColumns(targetTables.getColumns()); YCQLSelect select = new YCQLSelect(); - List> columns = new ArrayList<>(); + List columns = new ArrayList<>(); for (int i = 0; i < nrColumns; i++) { - Node expression = gen.generateExpression(); + YCQLExpression expression = gen.generateExpression(); columns.add(expression); } select.setFetchColumns(columns); List tables = targetTables.getTables(); - Optional> table = tables.stream() - .map(t -> new TableReferenceNode(t)).findFirst(); + Optional table = tables.stream().map(t -> new YCQLTableReference(t)).findFirst(); select.setFromList(table.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression()); diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java index 695301595..a6c855cf3 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.AbstractUpdateGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLErrors; @@ -47,7 +46,7 @@ private SQLQueryAdapter generate() { @Override protected void updateValue(YCQLColumn column) { - Node expr; + YCQLExpression expr; if (Randomly.getBooleanWithSmallProbability()) { expr = gen.generateExpression(); YCQLErrors.addExpressionErrors(errors); From 8f84c4eab86fc9df15a54fc1afd9a49d1d592183 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 14:33:52 +0800 Subject: [PATCH 478/856] [DataFusion] Update expressions to match new AST --- src/sqlancer/datafusion/DataFusionSchema.java | 3 +- .../datafusion/DataFusionToStringVisitor.java | 11 ++--- .../ast/DataFusionBinaryOperation.java | 11 +++++ .../ast/DataFusionColumnReference.java | 12 +++++ .../datafusion/ast/DataFusionConstant.java | 8 ++-- .../datafusion/ast/DataFusionFunction.java | 11 +++++ .../datafusion/ast/DataFusionJoin.java | 37 +++++++------- .../datafusion/ast/DataFusionSelect.java | 16 ++----- .../ast/DataFusionTableReference.java | 11 +++++ .../ast/DataFusionUnaryPostfixOperation.java | 11 +++++ .../ast/DataFusionUnaryPrefixOperation.java | 11 +++++ .../gen/DataFusionExpressionGenerator.java | 48 +++++++++---------- .../test/DataFusionQueryPartitioningBase.java | 5 +- 13 files changed, 122 insertions(+), 73 deletions(-) create mode 100644 src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionColumnReference.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionFunction.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionTableReference.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java create mode 100644 src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java diff --git a/src/sqlancer/datafusion/DataFusionSchema.java b/src/sqlancer/datafusion/DataFusionSchema.java index 4a5ed803a..b9f00a02e 100644 --- a/src/sqlancer/datafusion/DataFusionSchema.java +++ b/src/sqlancer/datafusion/DataFusionSchema.java @@ -13,7 +13,6 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.ast.newast.Node; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; @@ -123,7 +122,7 @@ public static DataFusionDataType parseFromDataFusionCatalog(String typeString) { } // TODO(datafusion) lots of hack here, should build our own Randomly later - public Node getRandomConstant(DataFusionGlobalState state) { + public DataFusionExpression getRandomConstant(DataFusionGlobalState state) { if (Randomly.getBooleanWithSmallProbability()) { return DataFusionConstant.createNullConstant(); } diff --git a/src/sqlancer/datafusion/DataFusionToStringVisitor.java b/src/sqlancer/datafusion/DataFusionToStringVisitor.java index 1f3030552..7d0d1b1b6 100644 --- a/src/sqlancer/datafusion/DataFusionToStringVisitor.java +++ b/src/sqlancer/datafusion/DataFusionToStringVisitor.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.ast.newast.NewToStringVisitor; -import sqlancer.common.ast.newast.Node; import sqlancer.datafusion.ast.DataFusionConstant; import sqlancer.datafusion.ast.DataFusionExpression; import sqlancer.datafusion.ast.DataFusionJoin; @@ -11,20 +10,20 @@ public class DataFusionToStringVisitor extends NewToStringVisitor { - public static String asString(Node expr) { + public static String asString(DataFusionExpression expr) { DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); visitor.visit(expr); return visitor.get(); } - public static String asString(List> exprs) { + public static String asString(List exprs) { DataFusionToStringVisitor visitor = new DataFusionToStringVisitor(); visitor.visit(exprs); return visitor.get(); } @Override - public void visitSpecific(Node expr) { + public void visitSpecific(DataFusionExpression expr) { if (expr instanceof DataFusionConstant) { visit((DataFusionConstant) expr); } else if (expr instanceof DataFusionSelect) { @@ -37,13 +36,13 @@ public void visitSpecific(Node expr) { } private void visit(DataFusionJoin join) { - visit(join.getLeftTable()); + visit((DataFusionExpression) join.getLeftTable()); sb.append(" "); sb.append(join.getJoinType()); sb.append(" "); sb.append(" JOIN "); - visit(join.getRightTable()); + visit((DataFusionExpression) join.getRightTable()); if (join.getOnCondition() != null) { sb.append(" ON "); visit(join.getOnCondition()); diff --git a/src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java b/src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java new file mode 100644 index 000000000..e59676be8 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class DataFusionBinaryOperation extends NewBinaryOperatorNode + implements DataFusionExpression { + public DataFusionBinaryOperation(DataFusionExpression left, DataFusionExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionColumnReference.java b/src/sqlancer/datafusion/ast/DataFusionColumnReference.java new file mode 100644 index 000000000..2391ef694 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionColumnReference.java @@ -0,0 +1,12 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.datafusion.DataFusionSchema; + +public class DataFusionColumnReference extends + ColumnReferenceNode implements DataFusionExpression { + public DataFusionColumnReference(DataFusionSchema.DataFusionColumn column) { + super(column); + } + +} diff --git a/src/sqlancer/datafusion/ast/DataFusionConstant.java b/src/sqlancer/datafusion/ast/DataFusionConstant.java index d123adb8a..90a997fcf 100644 --- a/src/sqlancer/datafusion/ast/DataFusionConstant.java +++ b/src/sqlancer/datafusion/ast/DataFusionConstant.java @@ -1,17 +1,15 @@ package sqlancer.datafusion.ast; -import sqlancer.common.ast.newast.Node; - -public class DataFusionConstant implements Node { +public class DataFusionConstant implements DataFusionExpression { private DataFusionConstant() { } - public static Node createIntConstant(long val) { + public static DataFusionExpression createIntConstant(long val) { return new DataFusionIntConstant(val); } - public static Node createNullConstant() { + public static DataFusionExpression createNullConstant() { return new DataFusionNullConstant(); } diff --git a/src/sqlancer/datafusion/ast/DataFusionFunction.java b/src/sqlancer/datafusion/ast/DataFusionFunction.java new file mode 100644 index 000000000..130fc04bf --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionFunction.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class DataFusionFunction extends NewFunctionNode implements DataFusionExpression { + public DataFusionFunction(List args, F func) { + super(args, func); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionJoin.java b/src/sqlancer/datafusion/ast/DataFusionJoin.java index 61cd9f556..430b2b812 100644 --- a/src/sqlancer/datafusion/ast/DataFusionJoin.java +++ b/src/sqlancer/datafusion/ast/DataFusionJoin.java @@ -5,8 +5,6 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; @@ -16,32 +14,29 @@ /* NOT IMPLEMENTED YET */ -public class DataFusionJoin implements Node { +public class DataFusionJoin implements DataFusionExpression { - private final TableReferenceNode leftTable; - private final TableReferenceNode rightTable; + private final DataFusionTableReference leftTable; + private final DataFusionTableReference rightTable; private final JoinType joinType; - private final Node onCondition; + private final DataFusionExpression onCondition; - public DataFusionJoin(TableReferenceNode leftTable, - TableReferenceNode rightTable, JoinType joinType, - Node whereCondition) { + public DataFusionJoin(DataFusionTableReference leftTable, DataFusionTableReference rightTable, JoinType joinType, + DataFusionExpression whereCondition) { this.leftTable = leftTable; this.rightTable = rightTable; this.joinType = joinType; this.onCondition = whereCondition; } - public static List> getJoins(List tables, - DataFusionGlobalState globalState) { + public static List getJoins(List tables, DataFusionGlobalState globalState) { // [t1_join_t2, t1_join_t3, ...] - List> tableList = tables.stream() - .map(t -> new TableReferenceNode(t)) + List tableList = tables.stream().map(t -> new DataFusionTableReference(t)) .collect(Collectors.toList()); - List> joinExpressions = new ArrayList<>(); + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - TableReferenceNode leftTable = tableList.remove(0); - TableReferenceNode rightTable = tableList.remove(0); + DataFusionTableReference leftTable = tableList.remove(0); + DataFusionTableReference rightTable = tableList.remove(0); List columns = new ArrayList<>(leftTable.getTable().getColumns()); columns.addAll(rightTable.getTable().getColumns()); // TODO(datafusion) this `joinGen` can generate super chaotic exprsions, maybe we should make it more like a @@ -59,16 +54,16 @@ public static List> getJoins(List ta return joinExpressions; } - public static DataFusionJoin createInnerJoin(TableReferenceNode left, - TableReferenceNode right, Node predicate) { + public static DataFusionJoin createInnerJoin(DataFusionTableReference left, DataFusionTableReference right, + DataFusionExpression predicate) { return new DataFusionJoin(left, right, JoinType.INNER, predicate); } - public TableReferenceNode getLeftTable() { + public DataFusionTableReference getLeftTable() { return leftTable; } - public TableReferenceNode getRightTable() { + public DataFusionTableReference getRightTable() { return rightTable; } @@ -76,7 +71,7 @@ public JoinType getJoinType() { return joinType; } - public Node getOnCondition() { + public DataFusionExpression getOnCondition() { return onCondition; } diff --git a/src/sqlancer/datafusion/ast/DataFusionSelect.java b/src/sqlancer/datafusion/ast/DataFusionSelect.java index a80758c4a..6fda25144 100644 --- a/src/sqlancer/datafusion/ast/DataFusionSelect.java +++ b/src/sqlancer/datafusion/ast/DataFusionSelect.java @@ -6,16 +6,12 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.Node; -import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema; -import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionTable; import sqlancer.datafusion.gen.DataFusionExpressionGenerator; -public class DataFusionSelect extends SelectBase> implements Node { +public class DataFusionSelect extends SelectBase implements DataFusionExpression { public Optional fetchColumnsString = Optional.empty(); // When available, override `fetchColumns` in base // class's `Node` representation (for display) public DataFusionExpressionGenerator exprGen; @@ -37,16 +33,14 @@ public static DataFusionSelect getRandomSelect(DataFusionGlobalState state) { // And generate a random expression which might contain those columns List randomColumns = DataFusionTable.getRandomColumns(randomTables); randomSelect.exprGen = new DataFusionExpressionGenerator(state).setColumns(randomColumns); - Node whereExpr = randomSelect.exprGen + DataFusionExpression whereExpr = randomSelect.exprGen .generateExpression(DataFusionSchema.DataFusionDataType.BOOLEAN); // Constructing result - List> randomTableNodes = randomTables.stream() - .map(t -> new TableReferenceNode(t)) - .collect(Collectors.toList()); - List> randomColumnNodes = randomColumns.stream() - .map((c) -> new ColumnReferenceNode(c)) + List randomTableNodes = randomTables.stream().map(t -> new DataFusionTableReference(t)) .collect(Collectors.toList()); + List randomColumnNodes = randomColumns.stream() + .map((c) -> new DataFusionColumnReference(c)).collect(Collectors.toList()); randomSelect.setFetchColumns(randomColumnNodes); randomSelect.setFromList(randomTableNodes); diff --git a/src/sqlancer/datafusion/ast/DataFusionTableReference.java b/src/sqlancer/datafusion/ast/DataFusionTableReference.java new file mode 100644 index 000000000..14445576a --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionTableReference.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.datafusion.DataFusionSchema; + +public class DataFusionTableReference extends TableReferenceNode + implements DataFusionExpression { + public DataFusionTableReference(DataFusionSchema.DataFusionTable table) { + super(table); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java b/src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java new file mode 100644 index 000000000..ba5629460 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionUnaryPostfixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class DataFusionUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements DataFusionExpression { + public DataFusionUnaryPostfixOperation(DataFusionExpression expr, BinaryOperatorNode.Operator op) { + super(expr, op); + } +} diff --git a/src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java b/src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java new file mode 100644 index 000000000..7109a12c2 --- /dev/null +++ b/src/sqlancer/datafusion/ast/DataFusionUnaryPrefixOperation.java @@ -0,0 +1,11 @@ +package sqlancer.datafusion.ast; + +import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class DataFusionUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements DataFusionExpression { + public DataFusionUnaryPrefixOperation(DataFusionExpression expr, BinaryOperatorNode.Operator operator) { + super(expr, operator); + } +} diff --git a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java index 3d7340615..88e486f47 100644 --- a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java @@ -12,22 +12,22 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; -import sqlancer.common.ast.newast.ColumnReferenceNode; -import sqlancer.common.ast.newast.NewBinaryOperatorNode; -import sqlancer.common.ast.newast.NewFunctionNode; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.ast.DataFusionBinaryOperation; +import sqlancer.datafusion.ast.DataFusionColumnReference; import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionFunction; +import sqlancer.datafusion.ast.DataFusionUnaryPostfixOperation; +import sqlancer.datafusion.ast.DataFusionUnaryPrefixOperation; import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; public final class DataFusionExpressionGenerator - extends TypedExpressionGenerator, DataFusionColumn, DataFusionDataType> { + extends TypedExpressionGenerator { private final DataFusionGlobalState globalState; @@ -51,7 +51,7 @@ protected boolean canGenerateColumnOfType(DataFusionDataType type) { } @Override - protected Node generateExpression(DataFusionDataType type, int depth) { + protected DataFusionExpression generateExpression(DataFusionDataType type, int depth) { if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { DataFusionDataType expectedType = type; if (Randomly.getBooleanWithRatherLowProbability()) { // ~10% @@ -85,8 +85,7 @@ protected Node generateExpression(DataFusionDataType type, argType = type; } - return new NewUnaryPrefixOperatorNode(generateExpression(argType, depth + 1), - randomExpr); + return new DataFusionUnaryPrefixOperation(generateExpression(argType, depth + 1), randomExpr); case UNARY_POSTFIX: dfAssert(randomExpr.argTypes.size() == 1 && randomExpr.nArgs == 1, "Unary expression should only have 1 argument" + randomExpr.argTypes); @@ -97,8 +96,7 @@ protected Node generateExpression(DataFusionDataType type, argType = type; } - return new NewUnaryPostfixOperatorNode(generateExpression(argType, depth + 1), - randomExpr); + return new DataFusionUnaryPostfixOperation(generateExpression(argType, depth + 1), randomExpr); case BINARY: dfAssert(randomExpr.argTypes.size() == 2 && randomExpr.nArgs == 2, "Binrary expression should only have 2 argument" + randomExpr.argTypes); @@ -119,7 +117,7 @@ protected Node generateExpression(DataFusionDataType type, } } - return new NewBinaryOperatorNode(generateExpression(argTypeList.get(0), depth + 1), + return new DataFusionBinaryOperation(generateExpression(argTypeList.get(0), depth + 1), generateExpression(argTypeList.get(1), depth + 1), randomExpr); case AGGREGATE: // Fall through @@ -133,13 +131,13 @@ protected Node generateExpression(DataFusionDataType type, return null; } - public Node generateFunctionExpression(DataFusionDataType type, int depth, + public DataFusionExpression generateFunctionExpression(DataFusionDataType type, int depth, DataFusionBaseExpr exprType) { if (exprType.isVariadic || Randomly.getBooleanWithSmallProbability()) { // TODO(datafusion) maybe add possible types. e.g. some function have signature variadic(INT/DOUBLE), then // only randomly pick from INT and DOUBLE int nArgs = Randomly.smallNumber(); // 0, 2, 4, ... smaller one is more likely - return new NewFunctionNode(generateExpressions(nArgs), exprType); + return new DataFusionFunction(generateExpressions(nArgs), exprType); } List funcArgTypeList = new ArrayList<>(); // types of current expression's input arguments @@ -161,13 +159,13 @@ public Node generateFunctionExpression(DataFusionDataType i++; } - List> argExpressions = new ArrayList<>(); + List argExpressions = new ArrayList<>(); for (DataFusionDataType dataType : funcArgTypeList) { argExpressions.add(generateExpression(dataType, depth + 1)); } - return new NewFunctionNode(argExpressions, exprType); + return new DataFusionFunction(argExpressions, exprType); } List filterColumns(DataFusionDataType type) { @@ -179,7 +177,7 @@ List filterColumns(DataFusionDataType type) { } @Override - protected Node generateColumn(DataFusionDataType type) { + protected DataFusionExpression generateColumn(DataFusionDataType type) { // HACK: if no col of such type exist, generate constant value instead List colsOfType = filterColumns(type); if (colsOfType.isEmpty()) { @@ -187,32 +185,32 @@ protected Node generateColumn(DataFusionDataType type) { } DataFusionColumn column = Randomly.fromList(colsOfType); - return new ColumnReferenceNode(column); + return new DataFusionColumnReference(column); } @Override - public Node generateConstant(DataFusionDataType type) { + public DataFusionExpression generateConstant(DataFusionDataType type) { return type.getRandomConstant(globalState); } @Override - public Node generatePredicate() { + public DataFusionExpression generatePredicate() { return generateExpression(DataFusionDataType.BOOLEAN, 0); } @Override - public Node negatePredicate(Node predicate) { - return new NewUnaryPrefixOperatorNode<>(predicate, createExpr(DataFusionBaseExprType.NOT)); + public DataFusionExpression negatePredicate(DataFusionExpression predicate) { + return new DataFusionUnaryPrefixOperation(predicate, createExpr(DataFusionBaseExprType.NOT)); } @Override - public Node isNull(Node expr) { - return new NewUnaryPostfixOperatorNode<>(expr, createExpr(DataFusionBaseExprType.IS_NULL)); + public DataFusionExpression isNull(DataFusionExpression expr) { + return new DataFusionUnaryPostfixOperation(expr, createExpr(DataFusionBaseExprType.IS_NULL)); } public static class DataFusionCastOperation extends NewUnaryPostfixOperatorNode { - public DataFusionCastOperation(Node expr, DataFusionDataType type) { + public DataFusionCastOperation(DataFusionExpression expr, DataFusionDataType type) { super(expr, new Operator() { @Override diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java index b304a4131..067b07677 100644 --- a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java +++ b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java @@ -2,7 +2,6 @@ import java.sql.SQLException; -import sqlancer.common.ast.newast.Node; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -12,7 +11,7 @@ import sqlancer.datafusion.gen.DataFusionExpressionGenerator; public class DataFusionQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase, DataFusionGlobalState> + extends TernaryLogicPartitioningOracleBase implements TestOracle { DataFusionGlobalState state; DataFusionExpressionGenerator gen; @@ -31,7 +30,7 @@ public void check() throws SQLException { } @Override - protected ExpressionGenerator> getGen() { + protected ExpressionGenerator getGen() { return gen; } From 5a8300438f732843059c73deb101f4873d6282e4 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 14:34:26 +0800 Subject: [PATCH 479/856] Remove Node interface --- src/sqlancer/common/ast/newast/Node.java | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 src/sqlancer/common/ast/newast/Node.java diff --git a/src/sqlancer/common/ast/newast/Node.java b/src/sqlancer/common/ast/newast/Node.java deleted file mode 100644 index 310e875c0..000000000 --- a/src/sqlancer/common/ast/newast/Node.java +++ /dev/null @@ -1,5 +0,0 @@ -package sqlancer.common.ast.newast; - -public interface Node { - -} From f36e1466414f94a0906ea79d8a85fbf93f68a720 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 27 Aug 2024 14:38:58 +0800 Subject: [PATCH 480/856] Fix linter errors --- src/sqlancer/databend/DatabendExpectedValueVisitor.java | 2 +- .../oracle/tlp/DorisQueryPartitioningAggregateTester.java | 4 ++-- src/sqlancer/mariadb/ast/MariaDBJoin.java | 1 - src/sqlancer/postgres/ast/PostgresJoin.java | 1 - src/sqlancer/sqlite3/ast/SQLite3Expression.java | 1 - 5 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/databend/DatabendExpectedValueVisitor.java b/src/sqlancer/databend/DatabendExpectedValueVisitor.java index e10ade97a..bfa6208a3 100644 --- a/src/sqlancer/databend/DatabendExpectedValueVisitor.java +++ b/src/sqlancer/databend/DatabendExpectedValueVisitor.java @@ -25,7 +25,7 @@ public class DatabendExpectedValueVisitor { private void print(DatabendExpression expr) { sb.append(DatabendToStringVisitor.asString(expr)); sb.append(" -- "); - sb.append((expr).getExpectedValue()); + sb.append(expr.getExpectedValue()); sb.append("\n"); } diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java index b3289e5ad..afc1dde43 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningAggregateTester.java @@ -16,8 +16,8 @@ import sqlancer.doris.DorisSchema.DorisCompositeDataType; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.ast.DorisAggregateOperation; -import sqlancer.doris.ast.DorisAlias; import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; +import sqlancer.doris.ast.DorisAlias; import sqlancer.doris.ast.DorisBinaryArithmeticOperation; import sqlancer.doris.ast.DorisBinaryOperation; import sqlancer.doris.ast.DorisCastOperation; @@ -26,8 +26,8 @@ import sqlancer.doris.ast.DorisFunction; import sqlancer.doris.ast.DorisSelect; import sqlancer.doris.ast.DorisUnaryPostfixOperation; -import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; +import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; import sqlancer.doris.visitor.DorisToStringVisitor; diff --git a/src/sqlancer/mariadb/ast/MariaDBJoin.java b/src/sqlancer/mariadb/ast/MariaDBJoin.java index d846f4605..f1d0892e1 100644 --- a/src/sqlancer/mariadb/ast/MariaDBJoin.java +++ b/src/sqlancer/mariadb/ast/MariaDBJoin.java @@ -36,7 +36,6 @@ public MariaDBTable getTable() { return table; } - @Override public MariaDBExpression getOnClause() { return onClause; } diff --git a/src/sqlancer/postgres/ast/PostgresJoin.java b/src/sqlancer/postgres/ast/PostgresJoin.java index 7bfef3f6f..ef6dfab1f 100644 --- a/src/sqlancer/postgres/ast/PostgresJoin.java +++ b/src/sqlancer/postgres/ast/PostgresJoin.java @@ -100,7 +100,6 @@ public PostgresExpression getRightTable() { return rightTable; } - @Override public PostgresExpression getOnClause() { return onClause; } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index d84f399a4..7b131b6b5 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -165,7 +165,6 @@ public SQLite3Table getTable() { return table; } - @Override public SQLite3Expression getOnClause() { return onClause; } From df613543ec194e74cc7e89023869e543bcdacb86 Mon Sep 17 00:00:00 2001 From: ming wei <43949290+malwaregarry@users.noreply.github.com> Date: Tue, 27 Aug 2024 14:59:11 +0800 Subject: [PATCH 481/856] Update readme with removed DBMSs (#964) * Update readme with removed DBMSs * Add reference PR for removed DBMBs --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index bb80d0794..b0fe4769b 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,16 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | | DataFusion | Preliminary | Typed | Only basic SQL features are supported. | +## Previously Supported DBMS + +Some DBMS were once supported but subsequently removed. + +| DBMS | Pull Request | Description | +| ---------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ArangoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because ArangoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. | # Using SQLancer From b55c7743f7a6e7e55fcedd71ee9d4031625ce5da Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 28 Aug 2024 17:00:01 +0800 Subject: [PATCH 482/856] [DuckDB] Implement NoREC expression generator interface --- src/sqlancer/duckdb/ast/DuckDBJoin.java | 6 +- src/sqlancer/duckdb/ast/DuckDBSelect.java | 26 ++++++- .../duckdb/gen/DuckDBExpressionGenerator.java | 76 ++++++++++++++++++- .../gen/DuckDBRandomQuerySynthesizer.java | 2 +- .../test/DuckDBQueryPartitioningBase.java | 2 +- 5 files changed, 105 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/duckdb/ast/DuckDBJoin.java b/src/sqlancer/duckdb/ast/DuckDBJoin.java index 82031d1e7..f677e9efc 100644 --- a/src/sqlancer/duckdb/ast/DuckDBJoin.java +++ b/src/sqlancer/duckdb/ast/DuckDBJoin.java @@ -66,8 +66,8 @@ public OuterType getOuterType() { return outerType; } - public static List getJoins(List tableList, DuckDBGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DuckDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { DuckDBTableReference leftTable = tableList.remove(0); DuckDBTableReference rightTable = tableList.remove(0); @@ -111,7 +111,7 @@ public static DuckDBJoin createInnerJoin(DuckDBTableReference left, DuckDBTableR return new DuckDBJoin(left, right, JoinType.INNER, predicate); } - public static DuckDBExpression createNaturalJoin(DuckDBTableReference left, DuckDBTableReference right, + public static DuckDBJoin createNaturalJoin(DuckDBTableReference left, DuckDBTableReference right, OuterType naturalJoinType) { DuckDBJoin join = new DuckDBJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); diff --git a/src/sqlancer/duckdb/ast/DuckDBSelect.java b/src/sqlancer/duckdb/ast/DuckDBSelect.java index 10d76fbbd..e18e57a4d 100644 --- a/src/sqlancer/duckdb/ast/DuckDBSelect.java +++ b/src/sqlancer/duckdb/ast/DuckDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.duckdb.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; +import sqlancer.duckdb.DuckDBSchema.DuckDBTable; +import sqlancer.duckdb.DuckDBToStringVisitor; -public class DuckDBSelect extends SelectBase implements DuckDBExpression { +public class DuckDBSelect extends SelectBase + implements Select, DuckDBExpression { private boolean isDistinct; @@ -14,4 +22,20 @@ public boolean isDistinct() { return isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DuckDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DuckDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DuckDBToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 49a390c07..000aa95d3 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -3,17 +3,22 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; +import sqlancer.duckdb.DuckDBSchema.DuckDBTable; +import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBBetweenOperator; import sqlancer.duckdb.ast.DuckDBBinaryOperator; import sqlancer.duckdb.ast.DuckDBCaseOperator; @@ -22,12 +27,18 @@ import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBFunction; import sqlancer.duckdb.ast.DuckDBInOperator; +import sqlancer.duckdb.ast.DuckDBJoin; import sqlancer.duckdb.ast.DuckDBOrderingTerm; +import sqlancer.duckdb.ast.DuckDBPostFixText; +import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; import sqlancer.duckdb.ast.DuckDBTernary; -public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator { +public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator + implements NoRECGenerator { private final DuckDBGlobalState globalState; + private List tables; public DuckDBExpressionGenerator(DuckDBGlobalState globalState) { this.globalState = globalState; @@ -437,4 +448,67 @@ public DuckDBExpression isNull(DuckDBExpression expr) { return new sqlancer.duckdb.ast.DuckDBUnaryPostfixOperator(expr, DuckDBUnaryPostfixOperator.IS_NULL); } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public DuckDBExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public DuckDBSelect generateSelect() { + return new DuckDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) + .collect(Collectors.toList()); + return DuckDBJoin.getJoins(tableList, globalState); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DuckDBTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DuckDBSelect select, DuckDBExpression whereCondition, + boolean shouldUseAggregate) { + List allColumns = columns.stream().map((c) -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); + if (shouldUseAggregate) { + DuckDBFunction aggr = new DuckDBFunction<>( + Arrays.asList(new DuckDBColumnReference( + new DuckDBColumn("*", new DuckDBCompositeDataType(DuckDBDataType.INT, 0), false, false))), + DuckDBAggregateFunction.COUNT); + select.setFetchColumns(Arrays.asList(aggr)); + } else { + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DuckDBSelect select, DuckDBExpression whereCondition) { + DuckDBExpression asText = new DuckDBPostFixText(new DuckDBCastOperation( + new DuckDBPostFixText(whereCondition, + " IS NOT NULL AND " + DuckDBToStringVisitor.asString(whereCondition)), + new DuckDBCompositeDataType(DuckDBDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } diff --git a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java index c4c0d7c95..d88d4f0b8 100644 --- a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java +++ b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java @@ -40,7 +40,7 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) .collect(Collectors.toList()); - List joins = DuckDBJoin.getJoins(tableList, globalState); + List joins = DuckDBJoin.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java index b4858df4c..1e75be7b3 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningBase.java @@ -46,7 +46,7 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) .collect(Collectors.toList()); - List joins = DuckDBJoin.getJoins(tableList, state); + List joins = DuckDBJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); From 9fc9a57aed4bc110771ba31d348ce6505e168d85 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 28 Aug 2024 17:00:29 +0800 Subject: [PATCH 483/856] [DuckDB] Use common NoREC oracle --- .../duckdb/gen/DuckDBExpressionGenerator.java | 4 +- .../duckdb/test/DuckDBNoRECOracle.java | 123 +++--------------- 2 files changed, 19 insertions(+), 108 deletions(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 000aa95d3..3f6abc37f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -471,7 +471,9 @@ public DuckDBSelect generateSelect() { public List getRandomJoinClauses() { List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) .collect(Collectors.toList()); - return DuckDBJoin.getJoins(tableList, globalState); + List joins = DuckDBJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; } @Override diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 63932492d..4c57c0920 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -1,135 +1,44 @@ package sqlancer.duckdb.test; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; -import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; -import sqlancer.duckdb.DuckDBSchema.DuckDBDataType; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.DuckDBSchema.DuckDBTables; -import sqlancer.duckdb.DuckDBToStringVisitor; -import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBJoin; -import sqlancer.duckdb.ast.DuckDBPostFixText; import sqlancer.duckdb.ast.DuckDBSelect; -import sqlancer.duckdb.ast.DuckDBTableReference; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBCastOperation; -public class DuckDBNoRECOracle extends NoRECBase implements TestOracle { +public class DuckDBNoRECOracle implements TestOracle { - private final DuckDBSchema s; + NoRECOracle oracle; public DuckDBNoRECOracle(DuckDBGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - DuckDBErrors.addExpressionErrors(errors); + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - DuckDBTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(state).setColumns(columns); - DuckDBExpression randomWhereCondition = gen.generateExpression(); - List tables = randomTables.getTables(); - List tableList = tables.stream().map(t -> new DuckDBTableReference(t)) - .collect(Collectors.toList()); - List joins = DuckDBJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(tableList.stream().collect(Collectors.toList()), randomWhereCondition, joins); - int firstCount = getFirstQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, - randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getSecondQuery(List tableList, DuckDBExpression randomWhereCondition, - List joins) throws SQLException { - DuckDBSelect select = new DuckDBSelect(); - // select.setGroupByClause(groupBys); - // DuckDBExpression isTrue = DuckDBPostfixOperation.create(randomWhereCondition, - // PostfixOperator.IS_TRUE); - DuckDBExpression asText = new DuckDBPostFixText(new DuckDBCastOperation( - new DuckDBPostFixText(randomWhereCondition, - " IS NOT NULL AND " + DuckDBToStringVisitor.asString(randomWhereCondition)), - new DuckDBCompositeDataType(DuckDBDataType.INT, 8)), "as count"); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(tableList); - // select.setSelectType(SelectType.ALL); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + DuckDBToStringVisitor.asString(select) + ") as res"; - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getFirstQueryCount(SQLConnection con, List tableList, List columns, - DuckDBExpression randomWhereCondition, List joins) throws SQLException { - DuckDBSelect select = new DuckDBSelect(); - // select.setGroupByClause(groupBys); - // DuckDBAggregate aggr = new DuckDBAggregate( - List allColumns = columns.stream().map((c) -> new DuckDBColumnReference(c)) - .collect(Collectors.toList()); - // DuckDBAggregateFunction.COUNT); - // select.setFetchColumns(Arrays.asList(aggr)); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new DuckDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - // select.setSelectType(SelectType.ALL); - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = DuckDBToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From 68835ed60bb09450a52533cfbf8b69ec88733720 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:11:19 +0800 Subject: [PATCH 484/856] Update the DuckDB version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9bee83485..129908302 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 0.4.0 + 1.0.0 com.facebook.presto From 0b228603e3e82f20a8d9c56fc83b101faf9d43cb Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:12:21 +0800 Subject: [PATCH 485/856] Disable partial indexes Consider the following SQL statements: CREATE TABLE t0(c0 INT); CREATE INDEX i0 on t0(c0) WHERE c0 > 0; This will result in an error "Not implemented Error: Creating partial indexes is not supported currently (exit status 1)". --- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index daa22da5f..312fe5828 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -8,8 +8,6 @@ import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.DuckDBToStringVisitor; -import sqlancer.duckdb.ast.DuckDBExpression; public final class DuckDBIndexGenerator { @@ -42,12 +40,6 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } } sb.append(")"); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - DuckDBExpression expr = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()) - .generateExpression(); - sb.append(DuckDBToStringVisitor.asString(expr)); - } errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("Cannot create an index on the rowid!"); From d421418ece6d4660f336de1d557a7cd72d073103 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:18:08 +0800 Subject: [PATCH 486/856] Change the expected error message for duplicate key constraint violations This affects statements such as the following: CREATE TABLE t0(c0 INT UNIQUE); INSERT INTO t0(c0) VALUES (0); INSERT INTO t0(c0) VALUES (0); Error: Constraint Error: Duplicate key "c0: 0" violates unique constraint. If this is an unexpected constraint violation please double check with the known index limitations section in our documentation (https://duckdb.org/docs/sql/indexes). (exit status 1) --- src/sqlancer/duckdb/DuckDBErrors.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 5c2909f4d..1cadb3a2a 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -114,7 +114,7 @@ public static List getInsertErrors() { errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); - errors.add("duplicate key"); + errors.add("Duplicate key"); errors.add("can't be cast because the value is out of range for the destination type"); errors.add("Could not convert string"); errors.add("Unimplemented type for cast"); From 3eda3423e14522b6fee942cbadc9b019d3105bb3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:32:32 +0800 Subject: [PATCH 487/856] Do not insert into or update the rowid See the example errors below CREATE TABLE t0(c0 INT); INSERT INTO t0 VALUES (0), (1); SELECT rowid FROM t0; -- 0, 1 INSERT INTO t0(rowid) VALUES (0); -- Binder Error: Table "t0" does not have a column with name "rowid" (exit status 1) UPDATE t0 SET rowid=5; -- Binder Error: Referenced update column rowid not found in table! (exit status 1) --- src/sqlancer/common/schema/AbstractTable.java | 4 ++++ src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java | 2 +- src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 4a57fa846..89d568867 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -83,6 +83,10 @@ public List getRandomNonEmptyColumnSubset() { return Randomly.nonEmptySubset(getColumns()); } + public List getRandomNonEmptyColumnSubsetFilter(Predicate predicate) { + return Randomly.nonEmptySubset(getColumns().stream().filter(predicate).collect(Collectors.toList())); + } + public List getRandomNonEmptyColumnSubset(int size) { return Randomly.nonEmptySubset(getColumns(), size); } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 4fd3f390c..6793d2b51 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -29,7 +29,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { private SQLQueryAdapter generate() { sb.append("INSERT INTO "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyColumnSubset(); + List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); sb.append(table.getName()); sb.append("("); sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index 3bb22b2ea..b4ffd0140 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -27,7 +27,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { private SQLQueryAdapter generate() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyColumnSubset(); + List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); sb.append("UPDATE "); sb.append(table.getName()); From 7701068472adf647236c00f1b2dd1aeed98d54b7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:38:39 +0800 Subject: [PATCH 488/856] Add an expected error for mismatched expression types in the CASE operator SELECT 1 BETWEEN 1.0 AND true; -- Binder Error: Cannot mix values of type DECIMAL(11,1) and BOOLEAN in BETWEEN clause - an explicit cast is required --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 1cadb3a2a..58b5eb110 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -63,6 +63,7 @@ public static List getExpressionErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); + errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); } private static List getRegexErrors() { From 7db4a6527576f8119ec493d4901bd9db60a02013 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:45:48 +0800 Subject: [PATCH 489/856] Add an expected error for mismatched expressions in CASE expressions SELECT CASE WHEN false THEN 5.0 ELSE true END; -- Binder Error: Cannot mix values of type BOOLEAN and DECIMAL(2,1) in CASE expression - an explicit cast is requiredSELECT CASE WHEN false THEN 5.0 ELSE true END; -- Binder Error: Cannot mix values of type BOOLEAN and DECIMAL(2,1) in CASE expression - an explicit cast is required --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 58b5eb110..4adce2b7b 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -64,6 +64,7 @@ public static List getExpressionErrors() { public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); + errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in CASE expression"); } private static List getRegexErrors() { From 2f78758f22f8b04527a5b2272756ad07f95edbb5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:51:19 +0800 Subject: [PATCH 490/856] Add an expected error for mismatch expression types in comparisons SELECT 1 >= ('a' || 'b'); -- Binder Error: Cannot compare values of type BIGINT and type VARCHAR - an explicit cast is required SELECT 1 >= 'a'; -- Conversion Error: Could not convert string 'a' to INT32 --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 4adce2b7b..5510d49e4 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -65,6 +65,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in CASE expression"); + errors.addRegexString("Cannot compare values of type .* and type .*"); } private static List getRegexErrors() { From 3d87be0fe5e7d66c63ad1674176ae48d6fcde09d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 22:56:22 +0800 Subject: [PATCH 491/856] Change the expected error message when creating unique indexes on duplicate data CREATE TABLE t0(c0 INT); INSERT INTO t0 VALUES (0); INSERT INTO t0 VALUES (0); CREATE UNIQUE INDEX i0 on t0(c0); -- Constraint Error: Data contains duplicates on indexed column(s) (exit status 1) --- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 312fe5828..80bbfd61c 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -19,7 +19,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { StringBuilder sb = new StringBuilder(); sb.append("CREATE "); if (Randomly.getBoolean()) { - errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); + errors.add("Data contains duplicates on indexed column(s)"); sb.append("UNIQUE "); } sb.append("INDEX "); From 3e513f2203db7be28bec7769541a2ba0ec57622d Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 23:18:53 +0800 Subject: [PATCH 492/856] Avoid assigning a rowid column to views CREATE VIEW v0 AS SELECT 1; SELECT rowid FROM v0; -- Binder Error: Referenced column "rowid" not found in FROM clause! --- src/sqlancer/common/schema/AbstractSchema.java | 4 ++++ src/sqlancer/duckdb/DuckDBErrors.java | 3 --- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractSchema.java b/src/sqlancer/common/schema/AbstractSchema.java index 1c66363e7..053c9540a 100644 --- a/src/sqlancer/common/schema/AbstractSchema.java +++ b/src/sqlancer/common/schema/AbstractSchema.java @@ -126,6 +126,10 @@ public String getFreeTableName() { } + public static boolean matchesViewName(String relationName) { + return relationName.startsWith("v"); + } + public String getFreeViewName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 5510d49e4..03f9c4b5a 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -46,9 +46,6 @@ public static List getExpressionErrors() { errors.add("Like pattern must not end with escape character!"); // LIKE - errors.add("does not have a column named \"rowid\""); // TODO: this can be removed if we can query whether a - // table supports rowids - errors.add("does not have a column named"); // TODO: this only happens for views whose underlying table has a // removed column errors.add("Contents of view were altered: types don't match!"); diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 6b94ce137..8e48bf89f 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -269,7 +269,7 @@ private static List getTableColumns(SQLConnection con, String tabl } } } - if (columns.stream().noneMatch(c -> c.isPrimaryKey())) { + if (columns.stream().noneMatch(c -> c.isPrimaryKey()) && !AbstractSchema.matchesViewName(tableName)) { // https://github.com/cwida/duckdb/issues/589 // https://github.com/cwida/duckdb/issues/588 // TODO: implement an option to enable/disable rowids From 993dd64c7676d1bdc28356fdfffd87ce178294ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 23:22:20 +0800 Subject: [PATCH 493/856] Add an expected error for mismatched expression types in the coalesce operator SELECT COALESCE(null, 'a', 5); -- Binder Error: Cannot mix values of type VARCHAR and INTEGER_LITERAL in COALESCE operator - an explicit cast is required (exit status 1) --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 03f9c4b5a..d3cc6f310 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -62,6 +62,7 @@ public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in CASE expression"); + errors.addRegexString("Cannot mix values of type .* and .* in COALESCE operator"); errors.addRegexString("Cannot compare values of type .* and type .*"); } From a2f3f0a8d49c4ccaa3fe1938aa5bced9d7078d22 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 23:31:30 +0800 Subject: [PATCH 494/856] Update expected regular expression errors SELECT 'a' SIMILAR TO 'a)'; -- Invalid Input Error: unexpected ): a) SELECT 'a' SIMILAR TO '[a'; -- Invalid Input Error: missing ]: [a --- src/sqlancer/duckdb/DuckDBErrors.java | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index d3cc6f310..29fc2f0fe 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -69,15 +69,8 @@ public static void addExpressionErrors(ExpectedErrors errors) { private static List getRegexErrors() { ArrayList errors = new ArrayList<>(); - errors.add("missing ]"); - errors.add("missing )"); - errors.add("invalid escape sequence"); - errors.add("no argument for repetition operator: "); - errors.add("bad repetition operator"); - errors.add("trailing \\"); - errors.add("invalid perl operator"); - errors.add("invalid character class range"); - errors.add("width is not integer"); + errors.add("Invalid Input Error:"); + errors.add("Invalid Input Error:"); return errors; } From fc156a0cce46180732b527181c758eba4cf6c40b Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 31 Aug 2024 23:32:22 +0800 Subject: [PATCH 495/856] Add another version for the GROUP BY clause error --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 29fc2f0fe..d7e0aadca 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -131,6 +131,7 @@ public static void addInsertErrors(ExpectedErrors errors) { public static List getGroupByErrors() { ArrayList errors = new ArrayList<>(); + errors.add("must appear in the GROUP BY clause or must be part of an aggregate function"); errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("GROUP BY term out of range"); From fdf853ff7cc48e0cf56a41ed0550f6ee54ea223c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 1 Sep 2024 00:05:06 +0800 Subject: [PATCH 496/856] Update the name of the NULL data type string for views --- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 8e48bf89f..379576fd8 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -203,7 +203,7 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { case "TIMESTAMP": primitiveType = DuckDBDataType.TIMESTAMP; break; - case "NULL": + case "\"NULL\"": primitiveType = DuckDBDataType.NULL; break; case "INTERVAL": From 8a935f18da5511f60cd381d6d591b19481a247b6 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 15:19:10 +0800 Subject: [PATCH 497/856] [TimescaleDB] Remove implementation --- src/sqlancer/Main.java | 2 - src/sqlancer/timescaledb/TimescaleDBBugs.java | 7 --- .../timescaledb/TimescaleDBGlobalState.java | 12 ----- .../timescaledb/TimescaleDBOptions.java | 27 ---------- .../timescaledb/TimescaleDBProvider.java | 30 ----------- .../timescaledb/TimescaleDBSchema.java | 52 ------------------- .../timescaledb/gen/TimescaleDBCommon.java | 20 ------- 7 files changed, 150 deletions(-) delete mode 100644 src/sqlancer/timescaledb/TimescaleDBBugs.java delete mode 100644 src/sqlancer/timescaledb/TimescaleDBGlobalState.java delete mode 100644 src/sqlancer/timescaledb/TimescaleDBOptions.java delete mode 100644 src/sqlancer/timescaledb/TimescaleDBProvider.java delete mode 100644 src/sqlancer/timescaledb/TimescaleDBSchema.java delete mode 100644 src/sqlancer/timescaledb/gen/TimescaleDBCommon.java diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 5ffc3e94c..46841095b 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -44,7 +44,6 @@ import sqlancer.questdb.QuestDBProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; -import sqlancer.timescaledb.TimescaleDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; import sqlancer.yugabyte.ysql.YSQLProvider; @@ -746,7 +745,6 @@ private static void checkForIssue799(List> providers) providers.add(new QuestDBProvider()); providers.add(new SQLite3Provider()); providers.add(new TiDBProvider()); - providers.add(new TimescaleDBProvider()); providers.add(new YCQLProvider()); providers.add(new YSQLProvider()); } diff --git a/src/sqlancer/timescaledb/TimescaleDBBugs.java b/src/sqlancer/timescaledb/TimescaleDBBugs.java deleted file mode 100644 index 079dfee74..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBBugs.java +++ /dev/null @@ -1,7 +0,0 @@ -package sqlancer.timescaledb; - -public final class TimescaleDBBugs { - private TimescaleDBBugs() { - - } -} diff --git a/src/sqlancer/timescaledb/TimescaleDBGlobalState.java b/src/sqlancer/timescaledb/TimescaleDBGlobalState.java deleted file mode 100644 index 519001b90..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBGlobalState.java +++ /dev/null @@ -1,12 +0,0 @@ -package sqlancer.timescaledb; - -import java.sql.SQLException; - -import sqlancer.postgres.PostgresGlobalState; - -public class TimescaleDBGlobalState extends PostgresGlobalState { - @Override - public TimescaleDBSchema readSchema() throws SQLException { - return TimescaleDBSchema.fromConnection(getConnection(), getDatabaseName()); - } -} diff --git a/src/sqlancer/timescaledb/TimescaleDBOptions.java b/src/sqlancer/timescaledb/TimescaleDBOptions.java deleted file mode 100644 index 03b479662..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBOptions.java +++ /dev/null @@ -1,27 +0,0 @@ -package sqlancer.timescaledb; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; - -import com.beust.jcommander.Parameter; - -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresOptions; -import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; - -public class TimescaleDBOptions extends PostgresOptions { - @Parameter(names = "--timescaledboracle", description = "Specifies which test oracle should be used for TimeScaleDB extension to PostgreSQL") - public List timescaleDBOracle = Arrays.asList(TimescaleDBOracleFactory.PQS); - - public enum TimescaleDBOracleFactory implements OracleFactory { - PQS { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresPivotedQuerySynthesisOracle(globalState); - } - }, - } -} diff --git a/src/sqlancer/timescaledb/TimescaleDBProvider.java b/src/sqlancer/timescaledb/TimescaleDBProvider.java deleted file mode 100644 index 756669128..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBProvider.java +++ /dev/null @@ -1,30 +0,0 @@ -package sqlancer.timescaledb; - -import com.google.auto.service.AutoService; - -import sqlancer.DatabaseProvider; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresOptions; -import sqlancer.postgres.PostgresProvider; -import sqlancer.timescaledb.gen.TimescaleDBCommon; - -@AutoService(DatabaseProvider.class) -public class TimescaleDBProvider extends PostgresProvider { - @SuppressWarnings("unchecked") - public TimescaleDBProvider() { - super((Class) (Object) TimescaleDBGlobalState.class, - (Class) (Object) TimescaleDBOptions.class); - } - - @Override - public String getDBMSName() { - return "timescaledb"; - } - - public static ExpectedErrors getTimescaleDBErrors() { - ExpectedErrors errors = new ExpectedErrors(); - TimescaleDBCommon.addTimescaleDBErrors(errors); - return errors; - } -} diff --git a/src/sqlancer/timescaledb/TimescaleDBSchema.java b/src/sqlancer/timescaledb/TimescaleDBSchema.java deleted file mode 100644 index 0c09d579c..000000000 --- a/src/sqlancer/timescaledb/TimescaleDBSchema.java +++ /dev/null @@ -1,52 +0,0 @@ -package sqlancer.timescaledb; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.SQLIntegrityConstraintViolationException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.SQLConnection; -import sqlancer.postgres.PostgresSchema; - -public class TimescaleDBSchema extends PostgresSchema { - - public TimescaleDBSchema(List databaseTables, String databaseName) { - super(new ArrayList<>(databaseTables), databaseName); - } - - public static class TimescaleDBTable extends PostgresTable { - public TimescaleDBTable(String tableName, List columns, List indexes, - TableType tableType, List statistics, boolean isView, boolean isInsertable) { - super(tableName, columns, indexes, tableType, statistics, isView, isInsertable); - } - - public TimescaleDBTable(PostgresTable table) { - super(table.getName(), table.getColumns(), table.getIndexes(), table.getTableType(), table.getStatistics(), - table.isView(), table.isInsertable()); - } - } - - public static TimescaleDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { - PostgresSchema schema = PostgresSchema.fromConnection(con, databaseName); - List databaseTables = new ArrayList<>(); - try (Statement s = con.createStatement(); - ResultSet rs = s.executeQuery("SELECT table_name FROM information_schema.tables")) { - while (rs.next()) { - String tableName = rs.getString("table_name"); - - PostgresTable t = schema.getDatabaseTable(tableName); - if (t == null) { - continue; - } - - TimescaleDBTable table = new TimescaleDBTable(t); - databaseTables.add(table); - } - } catch (SQLIntegrityConstraintViolationException e) { - throw new AssertionError(e); - } - return new TimescaleDBSchema(databaseTables, databaseName); - } -} diff --git a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java b/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java deleted file mode 100644 index 69c4148de..000000000 --- a/src/sqlancer/timescaledb/gen/TimescaleDBCommon.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.timescaledb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.common.query.ExpectedErrors; - -public final class TimescaleDBCommon { - private TimescaleDBCommon() { - - } - - public static List getTimescaleDBErrors() { - return new ArrayList<>(); - } - - public static void addTimescaleDBErrors(ExpectedErrors errors) { - errors.addAll(getTimescaleDBErrors()); - } -} From 8a1cad34158d3dd42f81c570ea8ad2a64f44c418 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 2 Sep 2024 15:33:53 +0800 Subject: [PATCH 498/856] Remove duplicate line --- src/sqlancer/duckdb/DuckDBErrors.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index d7e0aadca..1cc7bd96b 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -70,8 +70,6 @@ private static List getRegexErrors() { ArrayList errors = new ArrayList<>(); errors.add("Invalid Input Error:"); - errors.add("Invalid Input Error:"); - return errors; } From 98d1284b90933180cee461750b1028e039dceeaa Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 16:25:06 +0800 Subject: [PATCH 499/856] [CockroachDB] Implement NoREC expression generator interface --- .../ast/CockroachDBExpression.java | 5 +- .../cockroachdb/ast/CockroachDBJoin.java | 9 +- .../cockroachdb/ast/CockroachDBSelect.java | 27 +++++- .../gen/CockroachDBExpressionGenerator.java | 95 ++++++++++++++++++- .../oracle/CockroachDBCERTOracle.java | 2 +- 5 files changed, 130 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java b/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java index 555fb97fd..d0ac07310 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBExpression.java @@ -1,5 +1,8 @@ package sqlancer.cockroachdb.ast; -public interface CockroachDBExpression { +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.common.ast.newast.Expression; + +public interface CockroachDBExpression extends Expression { } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java index b53c4f681..a8cba22f9 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBJoin.java @@ -3,8 +3,12 @@ import java.util.Arrays; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.common.ast.newast.Join; -public class CockroachDBJoin implements CockroachDBExpression { +public class CockroachDBJoin + implements CockroachDBExpression, Join { private final CockroachDBExpression leftTable; private final CockroachDBExpression rightTable; @@ -49,7 +53,8 @@ public JoinType getJoinType() { return joinType; } - public void setOnCondition(CockroachDBExpression onCondition) { + @Override + public void setOnClause(CockroachDBExpression onCondition) { this.onCondition = onCondition; } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java index 2c7d33ff4..c043a9207 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.cockroachdb.ast; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; -public class CockroachDBSelect extends SelectBase implements CockroachDBExpression { +public class CockroachDBSelect extends SelectBase implements CockroachDBExpression, + Select { private boolean isDistinct; @@ -14,4 +22,21 @@ public void setDistinct(boolean isDistinct) { this.isDistinct = isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (CockroachDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (CockroachDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return CockroachDBVisitor.asString(this); + } + } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index 6326bcc95..d36e7673c 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -10,6 +10,8 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBCompositeDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.cockroachdb.CockroachDBVisitor; import sqlancer.cockroachdb.ast.CockroachDBAggregate; import sqlancer.cockroachdb.ast.CockroachDBAggregate.CockroachDBAggregateFunction; import sqlancer.cockroachdb.ast.CockroachDBBetweenOperation; @@ -29,6 +31,7 @@ import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBFunction; import sqlancer.cockroachdb.ast.CockroachDBInOperation; +import sqlancer.cockroachdb.ast.CockroachDBJoin; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonOperator; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonType; @@ -36,14 +39,20 @@ import sqlancer.cockroachdb.ast.CockroachDBOrderingTerm; import sqlancer.cockroachdb.ast.CockroachDBRegexOperation; import sqlancer.cockroachdb.ast.CockroachDBRegexOperation.CockroachDBRegexOperator; +import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.ast.CockroachDBTypeAnnotation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; -public class CockroachDBExpressionGenerator - extends TypedExpressionGenerator { +public class CockroachDBExpressionGenerator extends + TypedExpressionGenerator implements + NoRECGenerator { + private List tables; private final CockroachDBGlobalState globalState; public CockroachDBExpressionGenerator(CockroachDBGlobalState globalState) { @@ -84,7 +93,8 @@ public List getOrderingTerms() { @Override public CockroachDBExpression generateExpression(CockroachDBCompositeDataType type, int depth) { - // if (type == CockroachDBDataType.FLOAT && Randomly.getBooleanWithRatherLowProbability()) { + // if (type == CockroachDBDataType.FLOAT && + // Randomly.getBooleanWithRatherLowProbability()) { // type = CockroachDBDataType.INT; // } if (allowAggregates && Randomly.getBoolean()) { @@ -358,4 +368,83 @@ public CockroachDBExpression isNull(CockroachDBExpression expr) { return new CockroachDBUnaryPostfixOperation(expr, CockroachDBUnaryPostfixOperator.IS_NULL); } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public CockroachDBExpression generateBooleanExpression() { + return generateExpression(CockroachDBDataType.BOOL.get()); + } + + @Override + public CockroachDBSelect generateSelect() { + return new CockroachDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinExpressions = new ArrayList<>(); + List tableReferences = tables.stream().map(t -> new CockroachDBTableReference(t)) + .collect(Collectors.toList()); + while (tableReferences.size() >= 2 && Randomly.getBoolean()) { + CockroachDBTableReference leftTable = tableReferences.remove(0); + CockroachDBTableReference rightTable = tableReferences.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + + tables = tableReferences.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joinExpressions; + } + + @Override + public List getTableRefs() { + List tableReferences = tables.stream().map(t -> new CockroachDBTableReference(t)) + .collect(Collectors.toList()); + + return CockroachDBCommon.getTableReferences(tableReferences); + } + + @Override + public String generateOptimizedQueryString(CockroachDBSelect select, CockroachDBExpression whereCondition, + boolean shouldUseAggregate) { + CockroachDBColumn c = new CockroachDBColumn("COUNT(*)", null, false, false); + select.setWhereClause(whereCondition); + if (shouldUseAggregate) { + CockroachDBAggregate aggr = new CockroachDBAggregate(CockroachDBAggregateFunction.COUNT, + List.of(new CockroachDBColumnReference(new CockroachDBColumn("*", + new CockroachDBCompositeDataType(CockroachDBDataType.INT, 0), false, false)))); + select.setFetchColumns(List.of(aggr)); + } else { + select.setFetchColumns(List.of(new CockroachDBColumnReference(c))); + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(getOrderingTerms()); + } + } + return CockroachDBVisitor.asString(select); + } + + @Override + public String generateUnoptimizedQueryString(CockroachDBSelect select, CockroachDBExpression whereCondition) { + List tableList = select.getFromList(); + List joinList = select.getJoinList(); + String fromString = tableList.stream().map(t -> ((CockroachDBTableReference) t).getTable().getName()) + .collect(Collectors.joining(", ")); + if (!tableList.isEmpty() && !joinList.isEmpty()) { + fromString += ", "; + } + return "SELECT SUM(count) FROM (SELECT CAST(" + CockroachDBVisitor.asString(whereCondition) + + " IS TRUE AS INT) as count FROM " + fromString + " " + + joinList.stream().map(j -> CockroachDBVisitor.asString(j)).collect(Collectors.joining(", ")) + ")"; + } } diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java index 010188431..86d5d75ce 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -128,7 +128,7 @@ protected boolean mutateJoin() { columns.addAll(((CockroachDBTableReference) join.getLeftTable()).getTable().getColumns()); columns.addAll(((CockroachDBTableReference) join.getRightTable()).getTable().getColumns()); CockroachDBExpressionGenerator joinGen2 = new CockroachDBExpressionGenerator(state).setColumns(columns); - join.setOnCondition(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); + join.setOnClause(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); } JoinType newJoinType = CockroachDBJoin.JoinType.INNER; From 4819e48209bc126947413b725584a377af98e68b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 16:26:02 +0800 Subject: [PATCH 500/856] [CockroachDB] Use common NoREC oracle --- .../oracle/CockroachDBNoRECOracle.java | 120 ++++-------------- 1 file changed, 24 insertions(+), 96 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java index 2bc1873a0..3d0422dfd 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java @@ -2,66 +2,52 @@ import java.sql.SQLException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.SQLGlobalState; -import sqlancer.cockroachdb.CockroachDBCommon; +import sqlancer.Reproducer; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.cockroachdb.CockroachDBSchema; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; -import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.cockroachdb.ast.CockroachDBColumnReference; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBJoin; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -public class CockroachDBNoRECOracle extends NoRECBase - implements TestOracle { +public class CockroachDBNoRECOracle implements TestOracle { - private CockroachDBExpressionGenerator gen; + NoRECOracle oracle; public CockroachDBNoRECOracle(CockroachDBGlobalState globalState) { - super(globalState); - CockroachDBErrors.addExpressionErrors(errors); - CockroachDBErrors.addTransactionErrors(errors); - errors.add("unable to vectorize execution plan"); // SET vectorize=experimental_always; - errors.add(" mismatched physical types at index"); // SET vectorize=experimental_always; + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with(CockroachDBErrors.getTransactionErrors()).with("unable to vectorize execution plan") // SET + // vectorize=experimental_always; + .with(" mismatched physical types at index") // SET vectorize=experimental_always; + .build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - CockroachDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); - List tableL = tables.getTables().stream().map(t -> new CockroachDBTableReference(t)) - .collect(Collectors.toList()); - List tableList = CockroachDBCommon.getTableReferences(tableL); - gen = new CockroachDBExpressionGenerator(state).setColumns(tables.getColumns()); - List joinExpressions = getJoins(tableList, state); - CockroachDBExpression whereCondition = gen.generateExpression(CockroachDBDataType.BOOL.get()); - int optimizableCount = getOptimizedResult(whereCondition, tableList, errors, joinExpressions); - if (optimizableCount == -1) { - throw new IgnoreMeException(); - } - int nonOptimizableCount = getNonOptimizedResult(whereCondition, tableList, errors, joinExpressions); - if (nonOptimizableCount == -1) { - throw new IgnoreMeException(); - } - if (optimizableCount != nonOptimizableCount) { - state.getState().getLocalState().log(optimizedQueryString + ";\n" + unoptimizedQueryString + ";"); - throw new AssertionError(CockroachDBVisitor.asString(whereCondition)); - } + oracle.check(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } public static List getJoins(List tableList, @@ -80,62 +66,4 @@ public static List getJoins(List t return joinExpressions; } - private int getOptimizedResult(CockroachDBExpression whereCondition, List tableList, - ExpectedErrors errors, List joinExpressions) throws SQLException { - CockroachDBSelect select = new CockroachDBSelect(); - CockroachDBColumn c = new CockroachDBColumn("COUNT(*)", null, false, false); - select.setFetchColumns(Arrays.asList(new CockroachDBColumnReference(c))); - select.setFromList(tableList); - select.setWhereClause(whereCondition); - select.setJoinList(joinExpressions); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.getOrderingTerms()); - } - String s = CockroachDBVisitor.asString(select); - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(s); - } - this.optimizedQueryString = s; - SQLQueryAdapter q = new SQLQueryAdapter(s, errors); - return getCount(state, q); - } - - private int getNonOptimizedResult(CockroachDBExpression whereCondition, List tableList, - ExpectedErrors errors, List joinList) throws SQLException { - String fromString = tableList.stream().map(t -> ((CockroachDBTableReference) t).getTable().getName()) - .collect(Collectors.joining(", ")); - if (!tableList.isEmpty() && !joinList.isEmpty()) { - fromString += ", "; - } - String s = "SELECT SUM(count) FROM (SELECT CAST(" + CockroachDBVisitor.asString(whereCondition) - + " IS TRUE AS INT) as count FROM " + fromString + " " - + joinList.stream().map(j -> CockroachDBVisitor.asString(j)).collect(Collectors.joining(", ")) + ")"; - if (state.getOptions().logEachSelect()) { - state.getLogger().writeCurrent(s); - } - this.unoptimizedQueryString = s; - SQLQueryAdapter q = new SQLQueryAdapter(s, errors); - return getCount(state, q); - } - - private int getCount(SQLGlobalState globalState, SQLQueryAdapter q) throws AssertionError { - int count = 0; - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs == null) { - return -1; - } - if (rs.next()) { - count = rs.getInt(1); - } - } catch (Exception e) { - throw new AssertionError(q.getQueryString(), e); - } - return count; - } - - @Override - public String getLastQueryString() { - return optimizedQueryString; - } - } From 2fe7b6b00846aba7e975522f67255e14bafbb427 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 17:16:42 +0800 Subject: [PATCH 501/856] [CockroachDB] Add bug 85230 --- src/sqlancer/cockroachdb/CockroachDBBugs.java | 3 +++ .../cockroachdb/gen/CockroachDBTruncateGenerator.java | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 55045f637..6c34bcac4 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -56,6 +56,9 @@ public final class CockroachDBBugs { // https://github.com/cockroachdb/cockroach/issues/88037 public static boolean bug88037; + // https://github.com/cockroachdb/cockroach/issues/85230 + public static boolean bug85230 = true; + private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java index 904ee3a9f..44979487b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTruncateGenerator.java @@ -1,6 +1,7 @@ package sqlancer.cockroachdb.gen; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -14,6 +15,9 @@ private CockroachDBTruncateGenerator() { public static SQLQueryAdapter truncate(CockroachDBGlobalState globalState) { ExpectedErrors errors = new ExpectedErrors(); errors.add("is referenced by foreign key"); + if (CockroachDBBugs.bug85230) { + errors.add("found in depended-on-by references, no such index in this relation"); + } StringBuilder sb = new StringBuilder(); sb.append("TRUNCATE"); From d18ce4c0d9e8aeb98b7bf801959806e1c2851de5 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 2 Sep 2024 17:39:49 +0800 Subject: [PATCH 502/856] [CockroachDB] Update tests --- .github/workflows/main.yml | 4 +++- .../cockroachdb/CockroachDBErrors.java | 2 ++ ...roachDB.java => TestCockroachDBNoREC.java} | 10 ++++------ test/sqlancer/dbms/TestCockroachDBTLP.java | 19 +++++++++++++++++++ test/sqlancer/dbms/TestConfig.java | 1 + 5 files changed, 29 insertions(+), 7 deletions(-) rename test/sqlancer/dbms/{TestCockroachDB.java => TestCockroachDBNoREC.java} (58%) create mode 100644 test/sqlancer/dbms/TestCockroachDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68b02c443..5b50754b3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -144,7 +144,9 @@ jobs: - name: Create SQLancer user run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests - run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDB test + run: | + COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBNoREC test + COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBTLP test cockroachdb-qpg: name: QPG Tests (CockroachDB) diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index b52df11bd..7a06fe51e 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -206,6 +206,8 @@ public static List getExpressionErrors() { errors.add("argument of OFFSET must be type int, not type decimal"); errors.add("ERROR: for SELECT DISTINCT, ORDER BY expressions must appear in select list"); + errors.add("incompatible IF expressions"); + errors.addAll(getArrayErrors()); errors.addAll(getComputedColumnErrors()); diff --git a/test/sqlancer/dbms/TestCockroachDB.java b/test/sqlancer/dbms/TestCockroachDBNoREC.java similarity index 58% rename from test/sqlancer/dbms/TestCockroachDB.java rename to test/sqlancer/dbms/TestCockroachDBNoREC.java index 5eb701e3b..bf1a51193 100644 --- a/test/sqlancer/dbms/TestCockroachDB.java +++ b/test/sqlancer/dbms/TestCockroachDBNoREC.java @@ -7,15 +7,13 @@ import sqlancer.Main; -public class TestCockroachDB { +public class TestCockroachDBNoREC { @Test - public void testMySQL() { - String cockroachDB = System.getenv("COCKROACHDB_AVAILABLE"); - boolean cockroachDBIsAvailable = cockroachDB != null && cockroachDB.equalsIgnoreCase("true"); - assumeTrue(cockroachDBIsAvailable); + public void testCockroachDBNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb" })); + "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "NOREC" })); } } diff --git a/test/sqlancer/dbms/TestCockroachDBTLP.java b/test/sqlancer/dbms/TestCockroachDBTLP.java new file mode 100644 index 000000000..f916844ed --- /dev/null +++ b/test/sqlancer/dbms/TestCockroachDBTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCockroachDBTLP { + + @Test + public void testCockroachDBTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 084de35e7..1845aa422 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -4,6 +4,7 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; From 67ee096fbb6fc57a1ea3b74cb13d3513cd609322 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 10:10:40 +0800 Subject: [PATCH 503/856] [Databend] Implement NoREC expression generator interface --- .../databend/ast/DatabendExpression.java | 4 +- src/sqlancer/databend/ast/DatabendJoin.java | 16 ++-- src/sqlancer/databend/ast/DatabendSelect.java | 26 +++++- .../gen/DatabendNewExpressionGenerator.java | 85 ++++++++++++++++++- .../gen/DatabendRandomQuerySynthesizer.java | 4 +- .../tlp/DatabendQueryPartitioningBase.java | 2 +- 6 files changed, 124 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendExpression.java b/src/sqlancer/databend/ast/DatabendExpression.java index d53952140..d71e64ed7 100644 --- a/src/sqlancer/databend/ast/DatabendExpression.java +++ b/src/sqlancer/databend/ast/DatabendExpression.java @@ -1,8 +1,10 @@ package sqlancer.databend.ast; +import sqlancer.common.ast.newast.Expression; +import sqlancer.databend.DatabendSchema.DatabendColumn; import sqlancer.databend.DatabendSchema.DatabendDataType; -public interface DatabendExpression { +public interface DatabendExpression extends Expression { default DatabendDataType getExpectedType() { return null; diff --git a/src/sqlancer/databend/ast/DatabendJoin.java b/src/sqlancer/databend/ast/DatabendJoin.java index b927eab92..9386c1fe1 100644 --- a/src/sqlancer/databend/ast/DatabendJoin.java +++ b/src/sqlancer/databend/ast/DatabendJoin.java @@ -4,6 +4,7 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; @@ -11,12 +12,12 @@ import sqlancer.databend.DatabendSchema.DatabendTable; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendJoin implements DatabendExpression { +public class DatabendJoin implements DatabendExpression, Join { private final DatabendTableReference leftTable; private final DatabendTableReference rightTable; private final JoinType joinType; - private final DatabendExpression onCondition; + private DatabendExpression onCondition; private OuterType outerType; public enum JoinType { @@ -67,9 +68,8 @@ public OuterType getOuterType() { return outerType; } - public static List getJoins(List tableList, - DatabendGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DatabendGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { DatabendTableReference leftTable = tableList.remove(0); DatabendTableReference rightTable = tableList.remove(0); @@ -116,11 +116,15 @@ public static DatabendJoin createInnerJoin(DatabendTableReference left, Databend return new DatabendJoin(left, right, JoinType.INNER, predicate); } - public static DatabendExpression createNaturalJoin(DatabendTableReference left, DatabendTableReference right, + public static DatabendJoin createNaturalJoin(DatabendTableReference left, DatabendTableReference right, OuterType naturalJoinType) { DatabendJoin join = new DatabendJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; } + @Override + public void setOnClause(DatabendExpression onClause) { + onCondition = onClause; + } } diff --git a/src/sqlancer/databend/ast/DatabendSelect.java b/src/sqlancer/databend/ast/DatabendSelect.java index 3b0854ede..adc75d1cc 100644 --- a/src/sqlancer/databend/ast/DatabendSelect.java +++ b/src/sqlancer/databend/ast/DatabendSelect.java @@ -1,8 +1,16 @@ package sqlancer.databend.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendToStringVisitor; -public class DatabendSelect extends SelectBase implements DatabendExpression { +public class DatabendSelect extends SelectBase + implements DatabendExpression, Select { private boolean isDistinct; @@ -14,4 +22,20 @@ public boolean isDistinct() { return isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DatabendExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DatabendJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DatabendToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index 19a45eaa2..a3b7e996e 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -9,12 +9,17 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendSchema.DatabendRowValue; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendAggregateOperation; import sqlancer.databend.ast.DatabendAggregateOperation.DatabendAggregateFunction; import sqlancer.databend.ast.DatabendBetweenOperation; @@ -24,21 +29,29 @@ import sqlancer.databend.ast.DatabendBinaryComparisonOperation.DatabendBinaryComparisonOperator; import sqlancer.databend.ast.DatabendBinaryLogicalOperation; import sqlancer.databend.ast.DatabendBinaryLogicalOperation.DatabendBinaryLogicalOperator; +import sqlancer.databend.ast.DatabendCastOperation; +import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendColumnValue; import sqlancer.databend.ast.DatabendConstant; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendInOperation; +import sqlancer.databend.ast.DatabendJoin; import sqlancer.databend.ast.DatabendLikeOperation; import sqlancer.databend.ast.DatabendOrderByTerm; +import sqlancer.databend.ast.DatabendPostFixText; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.ast.DatabendUnaryPostfixOperation; import sqlancer.databend.ast.DatabendUnaryPostfixOperation.DatabendUnaryPostfixOperator; import sqlancer.databend.ast.DatabendUnaryPrefixOperation; import sqlancer.databend.ast.DatabendUnaryPrefixOperation.DatabendUnaryPrefixOperator; public class DatabendNewExpressionGenerator - extends TypedExpressionGenerator { + extends TypedExpressionGenerator + implements NoRECGenerator { private final DatabendGlobalState globalState; + private List tables; private final int maxDepth; private boolean allowAggregateFunctions; @@ -248,7 +261,8 @@ DatabendExpression getLike(int depth, DatabendDataType dataType) { } public DatabendExpression generateExpressionWithExpectedResult(DatabendDataType type) { - // DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + // DatabendNewExpressionGenerator gen = new + // DatabendNewExpressionGenerator(globalState).setColumns(columns); // gen.setRowValue(rowValue); DatabendExpression expr; do { @@ -340,4 +354,71 @@ public DatabendExpression generateHavingClause() { return expression; } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public DatabendExpression generateBooleanExpression() { + return generateExpression(DatabendDataType.BOOLEAN); + } + + @Override + public DatabendSelect generateSelect() { + return new DatabendSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DatabendTableReference(t)) + .collect(Collectors.toList()); + List joins = DatabendJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DatabendTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DatabendSelect select, DatabendExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + DatabendExpression aggr = new DatabendAggregateOperation( + List.of(new DatabendColumnReference(new DatabendColumn("*", + new DatabendCompositeDataType(DatabendDataType.INT, 0), false, false))), + DatabendAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + List allColumns = columns.stream().map((c) -> new DatabendColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DatabendSelect select, DatabendExpression whereCondition) { + DatabendExpression asText = new DatabendPostFixText(new DatabendCastOperation( + new DatabendPostFixText(whereCondition, + " IS NOT NULL AND " + DatabendToStringVisitor.asString(whereCondition)), + new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); + select.setFetchColumns(List.of(asText)); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } + } diff --git a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java index 7b7d0e35b..875c5afee 100644 --- a/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java +++ b/src/sqlancer/databend/gen/DatabendRandomQuerySynthesizer.java @@ -51,8 +51,8 @@ public static DatabendSelect generateSelect(DatabendGlobalState globalState, int List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DatabendTableReference(t)) .collect(Collectors.toList()); - List joins = DatabendJoin.getJoins(tableList, globalState); - select.setJoinList(joins); + List joins = DatabendJoin.getJoins(tableList, globalState); + select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { select.setWhereClause(gen.generateExpression(DatabendSchema.DatabendDataType.BOOLEAN)); diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java index 50a02f0ff..611164902 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningBase.java @@ -62,7 +62,7 @@ public void check() throws SQLException { List tableList = tables.stream().map(t -> new DatabendTableReference(t)) .collect(Collectors.toList()); if (!DatabendBugs.bug9236) { - List joins = DatabendJoin.getJoins(tableList, state); + List joins = DatabendJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); } select.setFromList(tableList.stream().collect(Collectors.toList())); From f28934a01e7de4b73c872fc2e1c9542804b22e17 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 10:11:28 +0800 Subject: [PATCH 504/856] [Databend] Use common NoREC oracle --- .../databend/test/DatabendNoRECOracle.java | 141 ++---------------- 1 file changed, 16 insertions(+), 125 deletions(-) diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java index 54f17858c..4d0d29711 100644 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ b/src/sqlancer/databend/test/DatabendNoRECOracle.java @@ -1,154 +1,45 @@ package sqlancer.databend.test; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema; import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; -import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.DatabendSchema.DatabendTables; -import sqlancer.databend.DatabendToStringVisitor; -import sqlancer.databend.ast.DatabendCastOperation; -import sqlancer.databend.ast.DatabendColumnReference; import sqlancer.databend.ast.DatabendExpression; import sqlancer.databend.ast.DatabendJoin; -import sqlancer.databend.ast.DatabendPostFixText; import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.ast.DatabendTableReference; import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendNoRECOracle extends NoRECBase implements TestOracle { +public class DatabendNoRECOracle implements TestOracle { - private final DatabendSchema s; + NoRECOracle oracle; public DatabendNoRECOracle(DatabendGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - DatabendErrors.addExpressionErrors(errors); + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - DatabendTables randomTables = s.getRandomTableNonEmptyAndViewTables(); // 随机获得nr张表 - List columns = randomTables.getColumns(); - if (columns.isEmpty()) { - debugColumns(columns, randomTables); // 调试代码，可忽略 - } - DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state).setColumns(columns); - DatabendExpression randomWhereCondition = gen.generateExpression(DatabendDataType.BOOLEAN); // 生成随机where条件 - List tables = randomTables.getTables(); - List tableList = tables.stream().map(t -> new DatabendTableReference(t)) - .collect(Collectors.toList()); - List joins = DatabendJoin.getJoins(tableList, state); - int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), - randomWhereCondition, joins); - int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, - randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getUnoptimizedQueryCount(List tableList, DatabendExpression randomWhereCondition, - List joins) throws SQLException { - DatabendSelect select = new DatabendSelect(); - // select.setGroupByClause(groupBys); - DatabendExpression asText = new DatabendPostFixText(new DatabendCastOperation( - new DatabendPostFixText(randomWhereCondition, - " IS NOT NULL AND " + DatabendToStringVisitor.asString(randomWhereCondition)), - new DatabendCompositeDataType(DatabendDataType.INT, 8)), "as count"); - - select.setFetchColumns(List.of(asText)); - select.setFromList(tableList); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + DatabendToStringVisitor.asString(select) + ") as res"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(SQLConnection con, List tableList, - List columns, DatabendExpression randomWhereCondition, List joins) - throws SQLException { - DatabendSelect select = new DatabendSelect(); - // select.setGroupByClause(groupBys); - List allColumns = columns.stream().map((c) -> new DatabendColumnReference(c)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new DatabendNewExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = DatabendToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - void debugColumns(List columns, DatabendTables randomTables) { - DatabendTables test = new DatabendTables(s.getDatabaseTables()); - System.out.println(String.format("tables size: %d", test.getTables().size())); - for (DatabendTable table : test.getTables()) { - System.out.println(String.format("%s", table.getName())); - for (DatabendColumn column : table.getColumns()) { - System.out.println(String.format("%s %s", column.getName(), column.getType())); - } - System.out.println("------------------------"); - } - System.out.println("+++++++++++++++++++++++++++++"); - for (DatabendTable table : randomTables.getTables()) { - System.out.println(String.format("%s", table.getName())); - for (DatabendColumn column : table.getColumns()) { - System.out.println(String.format("%s %s", column.getName(), column.getType())); - } - System.out.println("------------------------"); - } - throw new AssertionError( - String.format("randomTables size: %d,column is empty", randomTables.getTables().size())); + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From 0f5157e8ff5d6385a07659b417164445df48caac Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 10:55:15 +0800 Subject: [PATCH 505/856] [Doris] Implement NoREC expression generator interfaces --- src/sqlancer/doris/ast/DorisExpression.java | 4 +- src/sqlancer/doris/ast/DorisJoin.java | 15 +++- src/sqlancer/doris/ast/DorisSelect.java | 26 +++++- .../gen/DorisNewExpressionGenerator.java | 85 ++++++++++++++++++- .../gen/DorisRandomQuerySynthesizer.java | 2 +- .../tlp/DorisQueryPartitioningBase.java | 2 +- 6 files changed, 125 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/doris/ast/DorisExpression.java b/src/sqlancer/doris/ast/DorisExpression.java index 559ecd8f3..ee5cc7a26 100644 --- a/src/sqlancer/doris/ast/DorisExpression.java +++ b/src/sqlancer/doris/ast/DorisExpression.java @@ -1,8 +1,10 @@ package sqlancer.doris.ast; +import sqlancer.common.ast.newast.Expression; import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; -public interface DorisExpression { +public interface DorisExpression extends Expression { default DorisSchema.DorisDataType getExpectedType() { return null; } diff --git a/src/sqlancer/doris/ast/DorisJoin.java b/src/sqlancer/doris/ast/DorisJoin.java index 7c7a35c18..c92555ec3 100644 --- a/src/sqlancer/doris/ast/DorisJoin.java +++ b/src/sqlancer/doris/ast/DorisJoin.java @@ -4,17 +4,19 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.gen.DorisNewExpressionGenerator; -public class DorisJoin implements DorisExpression { +public class DorisJoin implements DorisExpression, Join { private final DorisTableReference leftTable; private final DorisTableReference rightTable; private final JoinType joinType; - private final DorisExpression onCondition; + private DorisExpression onCondition; public enum JoinType { INNER, STRAIGHT, LEFT, RIGHT; @@ -48,8 +50,8 @@ public DorisExpression getOnCondition() { return onCondition; } - public static List getJoins(List tableList, DorisGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, DorisGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { DorisTableReference leftTable = tableList.remove(0); DorisTableReference rightTable = tableList.remove(0); @@ -99,4 +101,9 @@ public static DorisJoin createLeftOuterJoin(DorisTableReference left, DorisTable DorisExpression predicate) { return new DorisJoin(left, right, JoinType.LEFT, predicate); } + + @Override + public void setOnClause(DorisExpression onClause) { + onCondition = onClause; + } } diff --git a/src/sqlancer/doris/ast/DorisSelect.java b/src/sqlancer/doris/ast/DorisSelect.java index 6194a0498..4921e0688 100644 --- a/src/sqlancer/doris/ast/DorisSelect.java +++ b/src/sqlancer/doris/ast/DorisSelect.java @@ -1,9 +1,17 @@ package sqlancer.doris.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; +import sqlancer.doris.visitor.DorisToStringVisitor; -public class DorisSelect extends SelectBase implements DorisExpression { +public class DorisSelect extends SelectBase + implements DorisExpression, Select { public enum DorisSelectDistinctType { @@ -37,4 +45,20 @@ public boolean isDistinct() { || this.selectDistinctType == DorisSelectDistinctType.DISTINCTROW; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DorisExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DorisJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DorisToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java index 13fe028ca..7cea29f31 100644 --- a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -10,12 +10,16 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.doris.DorisBugs; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisCompositeDataType; import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.DorisSchema.DorisRowValue; +import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.ast.DorisAggregateOperation; import sqlancer.doris.ast.DorisAggregateOperation.DorisAggregateFunction; import sqlancer.doris.ast.DorisBetweenOperation; @@ -27,21 +31,29 @@ import sqlancer.doris.ast.DorisBinaryLogicalOperation.DorisBinaryLogicalOperator; import sqlancer.doris.ast.DorisCaseOperation; import sqlancer.doris.ast.DorisCastOperation; +import sqlancer.doris.ast.DorisColumnReference; import sqlancer.doris.ast.DorisColumnValue; import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisFunctionOperation.DorisFunction; import sqlancer.doris.ast.DorisInOperation; +import sqlancer.doris.ast.DorisJoin; import sqlancer.doris.ast.DorisLikeOperation; import sqlancer.doris.ast.DorisOrderByTerm; +import sqlancer.doris.ast.DorisPostfixText; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.ast.DorisUnaryPostfixOperation; import sqlancer.doris.ast.DorisUnaryPostfixOperation.DorisUnaryPostfixOperator; import sqlancer.doris.ast.DorisUnaryPrefixOperation; import sqlancer.doris.ast.DorisUnaryPrefixOperation.DorisUnaryPrefixOperator; +import sqlancer.doris.visitor.DorisToStringVisitor; -public class DorisNewExpressionGenerator extends TypedExpressionGenerator { +public class DorisNewExpressionGenerator extends TypedExpressionGenerator + implements NoRECGenerator { private final DorisGlobalState globalState; + private List tables; private final int maxDepth; private boolean allowAggregateFunctions; @@ -449,4 +461,75 @@ public DorisExpression generateHavingClause() { public void setAllowAggregateFunctions(boolean allowAggregateFunctions) { this.allowAggregateFunctions = allowAggregateFunctions; } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public DorisExpression generateBooleanExpression() { + return generateExpression(DorisDataType.BOOLEAN); + } + + @Override + public DorisSelect generateSelect() { + return new DorisSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DorisTableReference(t)) + .collect(Collectors.toList()); + List joins = DorisJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DorisTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DorisSelect select, DorisExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + DorisExpression aggr = new DorisAggregateOperation( + List.of(new DorisColumnReference( + new DorisColumn("*", new DorisCompositeDataType(DorisDataType.INT, 0), false, false))), + DorisAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + + } else { + List allColumns = columns.stream().map((c) -> new DorisColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + List constants = new ArrayList<>(); + constants.add(new DorisConstant.DorisIntConstant( + Randomly.smallNumber() % select.getFetchColumns().size() + 1)); + select.setOrderByClauses(constants); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DorisSelect select, DorisExpression whereCondition) { + DorisExpression asText = new DorisPostfixText(new DorisCastOperation( + new DorisPostfixText(whereCondition, + " IS NOT NULL AND " + DorisToStringVisitor.asString(whereCondition)), + new DorisCompositeDataType(DorisDataType.INT, 8)), "as count"); + select.setFetchColumns(Arrays.asList(asText)); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } diff --git a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java index 518a204a5..e1ec50eb2 100644 --- a/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java +++ b/src/sqlancer/doris/gen/DorisRandomQuerySynthesizer.java @@ -47,7 +47,7 @@ public static DorisSelect generateSelect(DorisGlobalState globalState, int nrCol List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DorisTableReference(t)) .collect(Collectors.toList()); - List joins = DorisJoin.getJoins(tableList, globalState); + List joins = DorisJoin.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java index b3d3a7a6b..553e7739a 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningBase.java @@ -57,7 +57,7 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new DorisTableReference(t)) .collect(Collectors.toList()); - List joins = DorisJoin.getJoins(tableList, state); + List joins = DorisJoin.getJoins(tableList, state); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); select.setWhereClause(null); From c43052382a204314eca00140771b71a027b99dfd Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 10:55:41 +0800 Subject: [PATCH 506/856] [Doris] Use common NoREC oracle --- .../doris/oracle/DorisNoRECOracle.java | 121 +++--------------- 1 file changed, 16 insertions(+), 105 deletions(-) diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java index 8840a299e..9ce94b952 100644 --- a/src/sqlancer/doris/oracle/DorisNoRECOracle.java +++ b/src/sqlancer/doris/oracle/DorisNoRECOracle.java @@ -1,133 +1,44 @@ package sqlancer.doris.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema; import sqlancer.doris.DorisSchema.DorisColumn; -import sqlancer.doris.DorisSchema.DorisCompositeDataType; -import sqlancer.doris.DorisSchema.DorisDataType; import sqlancer.doris.DorisSchema.DorisTable; -import sqlancer.doris.DorisSchema.DorisTables; -import sqlancer.doris.ast.DorisCastOperation; -import sqlancer.doris.ast.DorisColumnReference; -import sqlancer.doris.ast.DorisConstant; import sqlancer.doris.ast.DorisExpression; import sqlancer.doris.ast.DorisJoin; -import sqlancer.doris.ast.DorisPostfixText; import sqlancer.doris.ast.DorisSelect; -import sqlancer.doris.ast.DorisTableReference; import sqlancer.doris.gen.DorisNewExpressionGenerator; -import sqlancer.doris.visitor.DorisToStringVisitor; -public class DorisNoRECOracle extends NoRECBase implements TestOracle { +public class DorisNoRECOracle implements TestOracle { - private final DorisSchema s; + NoRECOracle oracle; public DorisNoRECOracle(DorisGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - DorisErrors.addExpressionErrors(errors); - DorisErrors.addInsertErrors(errors); + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - DorisTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state).setColumns(columns); - DorisExpression randomWhereCondition = gen.generateExpression(DorisDataType.BOOLEAN); - List tables = randomTables.getTables(); - List tableList = tables.stream().map(t -> new DorisTableReference(t)) - .collect(Collectors.toList()); - List joins = DorisJoin.getJoins(tableList, state); - int secondCount = getUnoptimizedQueryCount(tableList.stream().collect(Collectors.toList()), - randomWhereCondition, joins); - int firstCount = getOptimizedQueryCount(con, tableList.stream().collect(Collectors.toList()), columns, - randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getUnoptimizedQueryCount(List tableList, DorisExpression randomWhereCondition, - List joins) throws SQLException { - DorisSelect select = new DorisSelect(); - DorisExpression asText = new DorisPostfixText(new DorisCastOperation( - new DorisPostfixText(randomWhereCondition, - " IS NOT NULL AND " + DorisToStringVisitor.asString(randomWhereCondition)), - new DorisCompositeDataType(DorisDataType.INT, 8)), "as count"); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(tableList); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + DorisToStringVisitor.asString(select) + ") as res"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getOptimizedQueryCount(SQLConnection con, List tableList, List columns, - DorisExpression randomWhereCondition, List joins) throws SQLException { - DorisSelect select = new DorisSelect(); - // select.setGroupByClause(groupBys); - List allColumns = columns.stream().map((c) -> new DorisColumnReference(c)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - List constants = new ArrayList<>(); - constants.add( - new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByClauses(constants); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = DorisToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From 3fc5c0c28956337a4f882bba2b7fb0899b40e2f7 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 11:36:41 +0800 Subject: [PATCH 507/856] [YSQL] Implement NoREC expression generator interfaces --- .../yugabyte/ysql/ast/YSQLExpression.java | 4 +- src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java | 11 +- .../yugabyte/ysql/ast/YSQLSelect.java | 12 +- .../ysql/gen/YSQLExpressionGenerator.java | 118 +++++++++++++++++- .../yugabyte/ysql/oracle/tlp/YSQLTLPBase.java | 55 ++++---- 5 files changed, 162 insertions(+), 38 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java index 9544c8838..5f68afd5d 100644 --- a/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLExpression.java @@ -1,8 +1,10 @@ package sqlancer.yugabyte.ysql.ast; +import sqlancer.common.ast.newast.Expression; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; -public interface YSQLExpression { +public interface YSQLExpression extends Expression { default YSQLDataType getExpressionType() { return null; diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java index 746d46cfd..4edcbd11a 100644 --- a/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLJoin.java @@ -1,12 +1,15 @@ package sqlancer.yugabyte.ysql.ast; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; -public class YSQLJoin implements YSQLExpression { +public class YSQLJoin implements YSQLExpression, Join { private final YSQLExpression tableReference; - private final YSQLExpression onClause; + private YSQLExpression onClause; private final YSQLJoinType type; public YSQLJoin(YSQLExpression tableReference, YSQLExpression onClause, YSQLJoinType type) { @@ -46,4 +49,8 @@ public static YSQLJoinType getRandom() { } + @Override + public void setOnClause(YSQLExpression onClause) { + this.onClause = onClause; + } } diff --git a/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java index ce477c238..fbf96ee45 100644 --- a/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java +++ b/src/sqlancer/yugabyte/ysql/ast/YSQLSelect.java @@ -5,10 +5,14 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLVisitor; -public class YSQLSelect extends SelectBase implements YSQLExpression { +public class YSQLSelect extends SelectBase + implements YSQLExpression, Select { private SelectType selectOption = SelectType.ALL; private List joinClauses = Collections.emptyList(); @@ -32,10 +36,12 @@ public YSQLDataType getExpressionType() { return null; } + @Override public List getJoinClauses() { return joinClauses; } + @Override public void setJoinClauses(List joinStatements) { this.joinClauses = joinStatements; @@ -132,4 +138,8 @@ public YSQLDataType getExpressionType() { } } + @Override + public String asString() { + return YSQLVisitor.asString(this); + } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java index c6a554d67..759d4d3ba 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java @@ -11,13 +11,18 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.yugabyte.ysql.YSQLCompoundDataType; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLProvider; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLRowValue; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; import sqlancer.yugabyte.ysql.ast.YSQLAggregate; +import sqlancer.yugabyte.ysql.ast.YSQLAggregate.YSQLAggregateFunction; import sqlancer.yugabyte.ysql.ast.YSQLBetweenOperation; import sqlancer.yugabyte.ysql.ast.YSQLBinaryArithmeticOperation; import sqlancer.yugabyte.ysql.ast.YSQLBinaryBitOperation; @@ -32,13 +37,17 @@ import sqlancer.yugabyte.ysql.ast.YSQLFunction; import sqlancer.yugabyte.ysql.ast.YSQLFunctionWithUnknownResult; import sqlancer.yugabyte.ysql.ast.YSQLInOperation; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; import sqlancer.yugabyte.ysql.ast.YSQLOrderByTerm; import sqlancer.yugabyte.ysql.ast.YSQLPOSIXRegularExpression; import sqlancer.yugabyte.ysql.ast.YSQLPostfixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; import sqlancer.yugabyte.ysql.ast.YSQLPrefixOperation; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; -public class YSQLExpressionGenerator implements ExpressionGenerator { +public class YSQLExpressionGenerator implements ExpressionGenerator, + NoRECGenerator { private final int maxDepth; @@ -46,6 +55,7 @@ public class YSQLExpressionGenerator implements ExpressionGenerator functionsAndTypes; private final List allowedFunctionTypes; private List columns; + private List tables; private YSQLRowValue rw; private boolean expectedResult; private YSQLGlobalState globalState; @@ -560,4 +570,110 @@ private enum IntExpression { UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION } + public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState, String name, YSQLTables tables) { + List columns = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + columns.add(gen.generateExpression(0)); + } + YSQLSelect select = new YSQLSelect(); + select.setFromList(tables.getTables().stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList())); + select.setFetchColumns(columns); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setOrderByClauses(gen.generateOrderBy()); + } + if (Randomly.getBoolean()) { + select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + if (Randomly.getBoolean()) { + select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); + } + } + if (Randomly.getBooleanWithRatherLowProbability()) { + select.setForClause(YSQLSelect.ForClause.getRandom()); + } + return new YSQLSelect.YSQLSubquery(select, name); + } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public YSQLExpression generateBooleanExpression() { + return generateExpression(YSQLDataType.BOOLEAN); + } + + @Override + public YSQLSelect generateSelect() { + return new YSQLSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLTable table = Randomly.fromList(tables); + tables.remove(table); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLSelect.YSQLSubquery subquery = createSubquery(globalState, String.format("sub%d", i), subqueryTables); + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(YSQLSelect select, YSQLExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + YSQLAggregate aggr = new YSQLAggregate(List.of(new YSQLColumnValue(YSQLColumn.createDummy("*"), null)), + YSQLAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + YSQLColumnValue allColumns = new YSQLColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(Arrays.asList(allColumns)); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBy()); + } + select.setWhereClause(whereCondition); + } + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(YSQLSelect select, YSQLExpression whereCondition) { + YSQLCastOperation isTrue = new YSQLCastOperation(whereCondition, YSQLCompoundDataType.create(YSQLDataType.INT)); + YSQLPostfixText asText = new YSQLPostfixText(isTrue, " as count", null, YSQLDataType.INT); + select.setFetchColumns(Collections.singletonList(asText)); + select.setSelectType(YSQLSelect.SelectType.ALL); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java index fa63ffc1f..57fe246e3 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPBase.java @@ -18,12 +18,10 @@ import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; -import sqlancer.yugabyte.ysql.ast.YSQLConstant; import sqlancer.yugabyte.ysql.ast.YSQLExpression; import sqlancer.yugabyte.ysql.ast.YSQLJoin; import sqlancer.yugabyte.ysql.ast.YSQLSelect; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; -import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; public class YSQLTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { @@ -39,34 +37,6 @@ public YSQLTLPBase(YSQLGlobalState state) { YSQLErrors.addCommonFetchErrors(errors); } - public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState, String name, YSQLTables tables) { - List columns = new ArrayList<>(); - YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(tables.getColumns()); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - columns.add(gen.generateExpression(0)); - } - YSQLSelect select = new YSQLSelect(); - select.setFromList(tables.getTables().stream().map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())) - .collect(Collectors.toList())); - select.setFetchColumns(columns); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - if (Randomly.getBoolean()) { - select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - if (Randomly.getBoolean()) { - select.setOffsetClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - } - } - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setForClause(YSQLSelect.ForClause.getRandom()); - } - return new YSQLSelect.YSQLSubquery(select, name); - } - @Override public void check() throws SQLException { s = state.getSchema(); @@ -76,10 +46,29 @@ public void check() throws SQLException { generateSelectBase(tables, joins); } - protected List getJoinStatements(YSQLGlobalState globalState, List columns, + public static List getJoinStatements(YSQLGlobalState globalState, List columns, List tables) { - return YSQLNoRECOracle.getJoinStatements(state, columns, tables); - // TODO joins + List joinStatements = new ArrayList<>(); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLTable table = Randomly.fromList(tables); + tables.remove(table); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + YSQLSelect.YSQLSubquery subquery = YSQLExpressionGenerator.createSubquery(globalState, + String.format("sub%d", i), subqueryTables); + YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); + YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); + YSQLJoin j = new YSQLJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; } protected void generateSelectBase(List tables, List joins) { From 4b0c923b2362686968317af80b59277318c68f03 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 11:37:17 +0800 Subject: [PATCH 508/856] [YSQL] Use common NoREC oracle --- .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 153 ++---------------- 1 file changed, 16 insertions(+), 137 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java index 37a6491ad..70804ce40 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java @@ -1,165 +1,44 @@ package sqlancer.yugabyte.ysql.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.yugabyte.ysql.YSQLCompoundDataType; +import sqlancer.common.query.ExpectedErrors; import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; import sqlancer.yugabyte.ysql.YSQLSchema; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; -import sqlancer.yugabyte.ysql.YSQLSchema.YSQLDataType; import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; -import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTables; -import sqlancer.yugabyte.ysql.YSQLVisitor; -import sqlancer.yugabyte.ysql.ast.YSQLCastOperation; -import sqlancer.yugabyte.ysql.ast.YSQLColumnValue; import sqlancer.yugabyte.ysql.ast.YSQLExpression; import sqlancer.yugabyte.ysql.ast.YSQLJoin; -import sqlancer.yugabyte.ysql.ast.YSQLPostfixText; import sqlancer.yugabyte.ysql.ast.YSQLSelect; import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; -import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPBase; -public class YSQLNoRECOracle extends NoRECBase implements TestOracle { +public class YSQLNoRECOracle implements TestOracle { - private final YSQLSchema s; + NoRECOracle oracle; public YSQLNoRECOracle(YSQLGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - YSQLErrors.addCommonExpressionErrors(errors); - YSQLErrors.addCommonFetchErrors(errors); - } - - public static List getJoinStatements(YSQLGlobalState globalState, List columns, - List tables) { - List joinStatements = new ArrayList<>(); - YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); - YSQLTable table = Randomly.fromList(tables); - tables.remove(table); - YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); - YSQLJoin j = new YSQLJoin(new YSQLSelect.YSQLFromTable(table, Randomly.getBoolean()), joinClause, options); - joinStatements.add(j); - } - // JOIN subqueries - for (int i = 0; i < Randomly.smallNumber(); i++) { - YSQLTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); - YSQLSelect.YSQLSubquery subquery = YSQLTLPBase.createSubquery(globalState, String.format("sub%d", i), - subqueryTables); - YSQLExpression joinClause = gen.generateExpression(YSQLDataType.BOOLEAN); - YSQLJoin.YSQLJoinType options = YSQLJoin.YSQLJoinType.getRandom(); - YSQLJoin j = new YSQLJoin(subquery, joinClause, options); - joinStatements.add(j); - } - return joinStatements; + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) + .with(YSQLErrors.getCommonFetchErrors()).with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - YSQLTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - YSQLExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); - - List joinStatements = getJoinStatements(state, columns, tables); - List fromTables = tables.stream() - .map(t -> new YSQLSelect.YSQLFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } - } - - private YSQLExpression getRandomWhereCondition(List columns) { - return new YSQLExpressionGenerator(state).setColumns(columns).generateExpression(YSQLDataType.BOOLEAN); + oracle.check(); } - private int getUnoptimizedQueryCount(List fromTables, YSQLExpression randomWhereCondition, - List joinStatements) throws SQLException { - YSQLSelect select = new YSQLSelect(); - YSQLCastOperation isTrue = new YSQLCastOperation(randomWhereCondition, - YSQLCompoundDataType.create(YSQLDataType.INT)); - YSQLPostfixText asText = new YSQLPostfixText(isTrue, " as count", null, YSQLDataType.INT); - select.setFetchColumns(Collections.singletonList(asText)); - select.setFromList(fromTables); - select.setSelectType(YSQLSelect.SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + YSQLVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getOptimizedQueryCount(List randomTables, List columns, - YSQLExpression randomWhereCondition, List joinStatements) throws SQLException { - YSQLSelect select = new YSQLSelect(); - YSQLColumnValue allColumns = new YSQLColumnValue(Randomly.fromList(columns), null); - select.setFetchColumns(Arrays.asList(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new YSQLExpressionGenerator(state).setColumns(columns).generateOrderBy()); - } - select.setSelectType(YSQLSelect.SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = YSQLVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From 1af322b4aaf46fe16778a6ebfb18a67372faaa99 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 11:49:24 +0800 Subject: [PATCH 509/856] [Yugabyte] Update tests --- .github/workflows/main.yml | 7 ++++-- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestYCQL.java | 18 ++++++++++++++++ test/sqlancer/dbms/TestYSQLNoREC.java | 19 ++++++++++++++++ test/sqlancer/dbms/TestYSQLPQS.java | 19 ++++++++++++++++ test/sqlancer/dbms/TestYSQLTLP.java | 19 ++++++++++++++++ test/sqlancer/dbms/TestYugabyte.java | 31 --------------------------- 7 files changed, 81 insertions(+), 33 deletions(-) create mode 100644 test/sqlancer/dbms/TestYCQL.java create mode 100644 test/sqlancer/dbms/TestYSQLNoREC.java create mode 100644 test/sqlancer/dbms/TestYSQLPQS.java create mode 100644 test/sqlancer/dbms/TestYSQLTLP.java delete mode 100644 test/sqlancer/dbms/TestYugabyte.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68b02c443..83c98666c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -522,8 +522,11 @@ jobs: docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false sleep 5 - name: Run Tests - run: YUGABYTE_AVAILABLE=true mvn -Dtest=TestYugabyte test - + run: | + YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLNoREC test + YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLTLP test + YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLPQS test + YUGABYTE_AVAILABLE=true mvn -Dtest=TestYCQL test doris: name: DBMS Tests (Apache Doris) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 084de35e7..510a34dd0 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -8,6 +8,7 @@ public class TestConfig { public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; + public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { String value = System.getenv(key); diff --git a/test/sqlancer/dbms/TestYCQL.java b/test/sqlancer/dbms/TestYCQL.java new file mode 100644 index 000000000..1198e2376 --- /dev/null +++ b/test/sqlancer/dbms/TestYCQL.java @@ -0,0 +1,18 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestYCQL { + @Test + public void testYCQL() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "cassandra", "--password", "cassandra", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ycql")); + } +} diff --git a/test/sqlancer/dbms/TestYSQLNoREC.java b/test/sqlancer/dbms/TestYSQLNoREC.java new file mode 100644 index 000000000..51f0487cc --- /dev/null +++ b/test/sqlancer/dbms/TestYSQLNoREC.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestYSQLNoREC { + @Test + public void testYSQLNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ysql", "--oracle", "NOREC")); + } +} diff --git a/test/sqlancer/dbms/TestYSQLPQS.java b/test/sqlancer/dbms/TestYSQLPQS.java new file mode 100644 index 000000000..6a21b6cb2 --- /dev/null +++ b/test/sqlancer/dbms/TestYSQLPQS.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestYSQLPQS { + @Test + public void testYSQLPQS() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ysql", "--oracle", "PQS")); + } +} diff --git a/test/sqlancer/dbms/TestYSQLTLP.java b/test/sqlancer/dbms/TestYSQLTLP.java new file mode 100644 index 000000000..fdcdf372a --- /dev/null +++ b/test/sqlancer/dbms/TestYSQLTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestYSQLTLP { + @Test + public void testYSQLTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.YUGABYTE_ENV)); + assertEquals(0, + Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", + "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "ysql", "--oracle", "QUERY_PARTITIONING")); + } +} diff --git a/test/sqlancer/dbms/TestYugabyte.java b/test/sqlancer/dbms/TestYugabyte.java deleted file mode 100644 index 10387948b..000000000 --- a/test/sqlancer/dbms/TestYugabyte.java +++ /dev/null @@ -1,31 +0,0 @@ -package sqlancer.dbms; - -import org.junit.jupiter.api.Test; -import sqlancer.Main; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class TestYugabyte { - - String yugabyteAvailable = System.getenv("YUGABYTE_AVAILABLE"); - boolean yugabyteIsAvailable = yugabyteAvailable != null && yugabyteAvailable.equalsIgnoreCase("true"); - - @Test - public void testYCQL() { - assumeTrue(yugabyteIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "cassandra", "--password", "cassandra", "--num-threads", "1", "--num-queries", - TestConfig.NUM_QUERIES, "ycql")); - } - - @Test - public void testYSQL() { - assumeTrue(yugabyteIsAvailable); - assertEquals(0, - Main.executeMain("--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--username", - "yugabyte", "--password", "yugabyte", "--num-threads", "1", "--num-queries", - TestConfig.NUM_QUERIES, "ysql")); - } -} From 1ef2f90192eb3179dc6aa50458adb06eca536088 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 13:41:13 +0800 Subject: [PATCH 510/856] [Materialize] Implement NoREC expression generator interfaces --- .../ast/MaterializeExpression.java | 4 +- .../materialize/ast/MaterializeJoin.java | 12 ++- .../materialize/ast/MaterializeSelect.java | 12 ++- .../gen/MaterializeExpressionGenerator.java | 100 +++++++++++++++++- 4 files changed, 123 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/materialize/ast/MaterializeExpression.java b/src/sqlancer/materialize/ast/MaterializeExpression.java index 8a42ad3f6..70444d8ee 100644 --- a/src/sqlancer/materialize/ast/MaterializeExpression.java +++ b/src/sqlancer/materialize/ast/MaterializeExpression.java @@ -1,8 +1,10 @@ package sqlancer.materialize.ast; +import sqlancer.common.ast.newast.Expression; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; -public interface MaterializeExpression { +public interface MaterializeExpression extends Expression { default MaterializeDataType getExpressionType() { return null; diff --git a/src/sqlancer/materialize/ast/MaterializeJoin.java b/src/sqlancer/materialize/ast/MaterializeJoin.java index 967cdf420..ef8f1ac61 100644 --- a/src/sqlancer/materialize/ast/MaterializeJoin.java +++ b/src/sqlancer/materialize/ast/MaterializeJoin.java @@ -1,9 +1,13 @@ package sqlancer.materialize.ast; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; -public class MaterializeJoin implements MaterializeExpression { +public class MaterializeJoin + implements MaterializeExpression, Join { public enum MaterializeJoinType { INNER, LEFT, RIGHT, FULL, CROSS; @@ -14,8 +18,8 @@ public static MaterializeJoinType getRandom() { } + private MaterializeExpression onClause; private final MaterializeExpression tableReference; - private final MaterializeExpression onClause; private final MaterializeJoinType type; public MaterializeJoin(MaterializeExpression tableReference, MaterializeExpression onClause, @@ -47,4 +51,8 @@ public MaterializeConstant getExpectedValue() { throw new AssertionError(); } + @Override + public void setOnClause(MaterializeExpression onClause) { + this.onClause = onClause; + } } diff --git a/src/sqlancer/materialize/ast/MaterializeSelect.java b/src/sqlancer/materialize/ast/MaterializeSelect.java index cef761428..db5b85a2f 100644 --- a/src/sqlancer/materialize/ast/MaterializeSelect.java +++ b/src/sqlancer/materialize/ast/MaterializeSelect.java @@ -5,10 +5,14 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeVisitor; -public class MaterializeSelect extends SelectBase implements MaterializeExpression { +public class MaterializeSelect extends SelectBase implements MaterializeExpression, + Select { private SelectType selectOption = SelectType.ALL; private List joinClauses = Collections.emptyList(); @@ -111,11 +115,13 @@ public MaterializeDataType getExpressionType() { return null; } + @Override public void setJoinClauses(List joinStatements) { this.joinClauses = joinStatements; } + @Override public List getJoinClauses() { return joinClauses; } @@ -132,4 +138,8 @@ public ForClause getForClause() { return forClause; } + @Override + public String asString() { + return MaterializeVisitor.asString(this); + } } diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java index 1396052a2..f3af50038 100644 --- a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -11,12 +11,16 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.materialize.MaterializeCompoundDataType; import sqlancer.materialize.MaterializeGlobalState; import sqlancer.materialize.MaterializeProvider; import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; import sqlancer.materialize.MaterializeSchema.MaterializeRowValue; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.MaterializeSchema.MaterializeTables; import sqlancer.materialize.ast.MaterializeAggregate; import sqlancer.materialize.ast.MaterializeAggregate.MaterializeAggregateFunction; import sqlancer.materialize.ast.MaterializeBetweenOperation; @@ -36,6 +40,8 @@ import sqlancer.materialize.ast.MaterializeFunction.MaterializeFunctionWithResult; import sqlancer.materialize.ast.MaterializeFunctionWithUnknownResult; import sqlancer.materialize.ast.MaterializeInOperation; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; import sqlancer.materialize.ast.MaterializeLikeOperation; import sqlancer.materialize.ast.MaterializeOrderByTerm; import sqlancer.materialize.ast.MaterializeOrderByTerm.MaterializeOrder; @@ -43,10 +49,17 @@ import sqlancer.materialize.ast.MaterializePOSIXRegularExpression.POSIXRegex; import sqlancer.materialize.ast.MaterializePostfixOperation; import sqlancer.materialize.ast.MaterializePostfixOperation.PostfixOperator; +import sqlancer.materialize.ast.MaterializePostfixText; import sqlancer.materialize.ast.MaterializePrefixOperation; import sqlancer.materialize.ast.MaterializePrefixOperation.PrefixOperator; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; +import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; +import sqlancer.materialize.ast.MaterializeSelect.SelectType; +import sqlancer.materialize.oracle.tlp.MaterializeTLPBase; -public class MaterializeExpressionGenerator implements ExpressionGenerator { +public class MaterializeExpressionGenerator implements ExpressionGenerator, + NoRECGenerator { private final int maxDepth; @@ -54,6 +67,8 @@ public class MaterializeExpressionGenerator implements ExpressionGenerator columns; + private List tables; + private MaterializeRowValue rw; private boolean expectedResult; @@ -514,4 +529,87 @@ public MaterializeExpression isNull(MaterializeExpression expr) { return new MaterializePostfixOperation(expr, PostfixOperator.IS_NULL); } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public MaterializeExpression generateBooleanExpression() { + return generateExpression(MaterializeDataType.BOOLEAN); + } + + @Override + public MaterializeSelect generateSelect() { + return new MaterializeSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeTable table = Randomly.fromList(tables); + tables.remove(table); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, + options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); + joinStatements.add(j); + } + + return joinStatements; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new MaterializeFromTable(t, Randomly.getBoolean())) + .collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(MaterializeSelect select, MaterializeExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + MaterializeAggregate aggr = new MaterializeAggregate( + List.of(new MaterializeColumnValue(MaterializeColumn.createDummy("*"), null)), + MaterializeAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + MaterializeColumnValue allColumns = new MaterializeColumnValue(Randomly.fromList(columns), null); + select.setFetchColumns(List.of(allColumns)); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBy()); + } + select.setSelectType(SelectType.ALL); + } + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(MaterializeSelect select, MaterializeExpression whereCondition) { + MaterializeCastOperation isTrue = new MaterializeCastOperation(whereCondition, + MaterializeCompoundDataType.create(MaterializeDataType.INT)); + MaterializePostfixText asText = new MaterializePostfixText(isTrue, " as count", null, MaterializeDataType.INT); + select.setFetchColumns(List.of(asText)); + select.setSelectType(SelectType.ALL); + select.setWhereClause(null); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } From c0d4fe541ed1f1ed04b91bc01b0de5d25b0ad156 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 13:41:36 +0800 Subject: [PATCH 511/856] [Materialize] Use common NoREC oracle --- .../oracle/MaterializeNoRECOracle.java | 160 ++---------------- .../oracle/tlp/MaterializeTLPBase.java | 27 ++- 2 files changed, 39 insertions(+), 148 deletions(-) diff --git a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java index bf9cc43fd..e0f665d93 100644 --- a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java +++ b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java @@ -1,175 +1,45 @@ package sqlancer.materialize.oracle; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.materialize.MaterializeCompoundDataType; +import sqlancer.common.query.ExpectedErrors; import sqlancer.materialize.MaterializeGlobalState; import sqlancer.materialize.MaterializeSchema; import sqlancer.materialize.MaterializeSchema.MaterializeColumn; -import sqlancer.materialize.MaterializeSchema.MaterializeDataType; import sqlancer.materialize.MaterializeSchema.MaterializeTable; -import sqlancer.materialize.MaterializeSchema.MaterializeTables; -import sqlancer.materialize.MaterializeVisitor; -import sqlancer.materialize.ast.MaterializeCastOperation; -import sqlancer.materialize.ast.MaterializeColumnValue; import sqlancer.materialize.ast.MaterializeExpression; import sqlancer.materialize.ast.MaterializeJoin; -import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; -import sqlancer.materialize.ast.MaterializePostfixText; import sqlancer.materialize.ast.MaterializeSelect; -import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; -import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; -import sqlancer.materialize.ast.MaterializeSelect.SelectType; import sqlancer.materialize.gen.MaterializeCommon; import sqlancer.materialize.gen.MaterializeExpressionGenerator; -import sqlancer.materialize.oracle.tlp.MaterializeTLPBase; -public class MaterializeNoRECOracle extends NoRECBase - implements TestOracle { +public class MaterializeNoRECOracle implements TestOracle { - private final MaterializeSchema s; + NoRECOracle oracle; public MaterializeNoRECOracle(MaterializeGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - MaterializeCommon.addCommonExpressionErrors(errors); - MaterializeCommon.addCommonFetchErrors(errors); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) + .with(MaterializeCommon.getCommonFetchErrors()).with("canceling statement due to statement timeout") + .build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - MaterializeTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - MaterializeExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); - - List joinStatements = getJoinStatements(state, columns, tables); - List fromTables = tables.stream() - .map(t -> new MaterializeFromTable(t, Randomly.getBoolean())).collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, columns, randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } - } - - public static List getJoinStatements(MaterializeGlobalState globalState, - List columns, List tables) { - List joinStatements = new ArrayList<>(); - MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); - MaterializeTable table = Randomly.fromList(tables); - tables.remove(table); - MaterializeJoinType options = MaterializeJoinType.getRandom(); - MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, - options); - joinStatements.add(j); - } - // JOIN subqueries - for (int i = 0; i < Randomly.smallNumber(); i++) { - MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); - MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), - subqueryTables); - MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); - MaterializeJoinType options = MaterializeJoinType.getRandom(); - MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); - joinStatements.add(j); - } - return joinStatements; - } - - private MaterializeExpression getRandomWhereCondition(List columns) { - return new MaterializeExpressionGenerator(state).setColumns(columns) - .generateExpression(MaterializeDataType.BOOLEAN); - } - - private int getUnoptimizedQueryCount(List fromTables, - MaterializeExpression randomWhereCondition, List joinStatements) throws SQLException { - MaterializeSelect select = new MaterializeSelect(); - MaterializeCastOperation isTrue = new MaterializeCastOperation(randomWhereCondition, - MaterializeCompoundDataType.create(MaterializeDataType.INT)); - MaterializePostfixText asText = new MaterializePostfixText(isTrue, " as count", null, MaterializeDataType.INT); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(fromTables); - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + MaterializeVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + oracle.check(); } - private int getOptimizedQueryCount(List randomTables, List columns, - MaterializeExpression randomWhereCondition, List joinStatements) throws SQLException { - MaterializeSelect select = new MaterializeSelect(); - MaterializeColumnValue allColumns = new MaterializeColumnValue(Randomly.fromList(columns), null); - select.setFetchColumns(Arrays.asList(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new MaterializeExpressionGenerator(state).setColumns(columns).generateOrderBy()); - } - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = MaterializeVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } @Override public String getLastQueryString() { - return optimizedQueryString; + return oracle.getLastQueryString(); } } diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java index dcf16dfad..36711997a 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java @@ -20,13 +20,13 @@ import sqlancer.materialize.ast.MaterializeConstant; import sqlancer.materialize.ast.MaterializeExpression; import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeJoin.MaterializeJoinType; import sqlancer.materialize.ast.MaterializeSelect; import sqlancer.materialize.ast.MaterializeSelect.ForClause; import sqlancer.materialize.ast.MaterializeSelect.MaterializeFromTable; import sqlancer.materialize.ast.MaterializeSelect.MaterializeSubquery; import sqlancer.materialize.gen.MaterializeCommon; import sqlancer.materialize.gen.MaterializeExpressionGenerator; -import sqlancer.materialize.oracle.MaterializeNoRECOracle; public class MaterializeTLPBase extends TernaryLogicPartitioningOracleBase @@ -54,8 +54,29 @@ public void check() throws SQLException { protected List getJoinStatements(MaterializeGlobalState globalState, List columns, List tables) { - return MaterializeNoRECOracle.getJoinStatements(state, columns, tables); - // TODO joins + List joinStatements = new ArrayList<>(); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeTable table = Randomly.fromList(tables); + tables.remove(table); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(new MaterializeFromTable(table, Randomly.getBoolean()), joinClause, + options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + MaterializeTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + MaterializeSubquery subquery = MaterializeTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + MaterializeExpression joinClause = gen.generateExpression(MaterializeDataType.BOOLEAN); + MaterializeJoinType options = MaterializeJoinType.getRandom(); + MaterializeJoin j = new MaterializeJoin(subquery, joinClause, options); + joinStatements.add(j); + } + + return joinStatements; } protected void generateSelectBase(List tables, List joins) { From 7fddc4bce9c513dc5b47f1f44f2f34e7bec5f056 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 13:49:41 +0800 Subject: [PATCH 512/856] [Materialize] Update tests --- .github/workflows/main.yml | 3 ++- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestMaterializeNoREC.java | 21 +++++++++++++++++++ test/sqlancer/dbms/TestMaterializePQS.java | 5 +---- ...terialize.java => TestMaterializeTLP.java} | 9 +++----- 5 files changed, 28 insertions(+), 11 deletions(-) create mode 100644 test/sqlancer/dbms/TestMaterializeNoREC.java rename test/sqlancer/dbms/{TestMaterialize.java => TestMaterializeTLP.java} (67%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 42845c307..aaafeb8cd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -323,7 +323,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterialize + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeNoREC + MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeTLP MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializePQS materialize-qpg: diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 4ef95d907..0ed58e137 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -8,6 +8,7 @@ public class TestConfig { public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; + public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestMaterializeNoREC.java b/test/sqlancer/dbms/TestMaterializeNoREC.java new file mode 100644 index 000000000..e4ac980ce --- /dev/null +++ b/test/sqlancer/dbms/TestMaterializeNoREC.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMaterializeNoREC { + + @Test + public void test() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MATERIALIZE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--username", "materialize", + "materialize", "--oracle", "NOREC", "--set-max-tables-mvs", "true" })); + } + +} diff --git a/test/sqlancer/dbms/TestMaterializePQS.java b/test/sqlancer/dbms/TestMaterializePQS.java index 5d4d795e4..0c6be974d 100644 --- a/test/sqlancer/dbms/TestMaterializePQS.java +++ b/test/sqlancer/dbms/TestMaterializePQS.java @@ -9,12 +9,9 @@ public class TestMaterializePQS { - String materializeAvailable = System.getenv("MATERIALIZE_AVAILABLE"); - boolean materializeIsAvailable = materializeAvailable != null && materializeAvailable.equalsIgnoreCase("true"); - @Test public void test() { - assumeTrue(materializeIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MATERIALIZE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--random-string-generation", diff --git a/test/sqlancer/dbms/TestMaterialize.java b/test/sqlancer/dbms/TestMaterializeTLP.java similarity index 67% rename from test/sqlancer/dbms/TestMaterialize.java rename to test/sqlancer/dbms/TestMaterializeTLP.java index 1818aa513..a98b3e053 100644 --- a/test/sqlancer/dbms/TestMaterialize.java +++ b/test/sqlancer/dbms/TestMaterializeTLP.java @@ -7,18 +7,15 @@ import sqlancer.Main; -public class TestMaterialize { - - String materializeAvailable = System.getenv("MATERIALIZE_AVAILABLE"); - boolean materializeIsAvailable = materializeAvailable != null && materializeAvailable.equalsIgnoreCase("true"); +public class TestMaterializeTLP { @Test public void test() { - assumeTrue(materializeIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MATERIALIZE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--username", "materialize", - "materialize", "--set-max-tables-mvs", "true" })); + "materialize", "--set-max-tables-mvs", "true", "--oracle", "QUERY_PARTITIONING" })); } } From fd1ba0ebb7c2acabf1842b5367a02ffdb6e49581 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 14:58:11 +0800 Subject: [PATCH 513/856] [DataFusion] Implement NoREC expression generator interfaces --- .../datafusion/ast/DataFusionExpression.java | 5 +- .../datafusion/ast/DataFusionJoin.java | 18 +++-- .../datafusion/ast/DataFusionSelect.java | 23 +++++- .../gen/DataFusionExpressionGenerator.java | 79 ++++++++++++++++++- 4 files changed, 113 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/datafusion/ast/DataFusionExpression.java b/src/sqlancer/datafusion/ast/DataFusionExpression.java index 6da95cd93..eaf84ba1d 100644 --- a/src/sqlancer/datafusion/ast/DataFusionExpression.java +++ b/src/sqlancer/datafusion/ast/DataFusionExpression.java @@ -1,5 +1,8 @@ package sqlancer.datafusion.ast; -public interface DataFusionExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; + +public interface DataFusionExpression extends Expression { } diff --git a/src/sqlancer/datafusion/ast/DataFusionJoin.java b/src/sqlancer/datafusion/ast/DataFusionJoin.java index 430b2b812..05718c5ad 100644 --- a/src/sqlancer/datafusion/ast/DataFusionJoin.java +++ b/src/sqlancer/datafusion/ast/DataFusionJoin.java @@ -2,9 +2,9 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; @@ -14,12 +14,13 @@ /* NOT IMPLEMENTED YET */ -public class DataFusionJoin implements DataFusionExpression { +public class DataFusionJoin + implements DataFusionExpression, Join { private final DataFusionTableReference leftTable; private final DataFusionTableReference rightTable; private final JoinType joinType; - private final DataFusionExpression onCondition; + private DataFusionExpression onCondition; public DataFusionJoin(DataFusionTableReference leftTable, DataFusionTableReference rightTable, JoinType joinType, DataFusionExpression whereCondition) { @@ -29,11 +30,10 @@ public DataFusionJoin(DataFusionTableReference leftTable, DataFusionTableReferen this.onCondition = whereCondition; } - public static List getJoins(List tables, DataFusionGlobalState globalState) { + public static List getJoins(List tableList, + DataFusionGlobalState globalState) { // [t1_join_t2, t1_join_t3, ...] - List tableList = tables.stream().map(t -> new DataFusionTableReference(t)) - .collect(Collectors.toList()); - List joinExpressions = new ArrayList<>(); + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { DataFusionTableReference leftTable = tableList.remove(0); DataFusionTableReference rightTable = tableList.remove(0); @@ -84,4 +84,8 @@ public static JoinType getRandom() { } } + @Override + public void setOnClause(DataFusionExpression onClause) { + onCondition = onClause; + } } diff --git a/src/sqlancer/datafusion/ast/DataFusionSelect.java b/src/sqlancer/datafusion/ast/DataFusionSelect.java index 6fda25144..10cf4859f 100644 --- a/src/sqlancer/datafusion/ast/DataFusionSelect.java +++ b/src/sqlancer/datafusion/ast/DataFusionSelect.java @@ -6,12 +6,16 @@ import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionToStringVisitor; import sqlancer.datafusion.gen.DataFusionExpressionGenerator; -public class DataFusionSelect extends SelectBase implements DataFusionExpression { +public class DataFusionSelect extends SelectBase implements DataFusionExpression, + Select { public Optional fetchColumnsString = Optional.empty(); // When available, override `fetchColumns` in base // class's `Node` representation (for display) public DataFusionExpressionGenerator exprGen; @@ -58,4 +62,21 @@ public static DataFusionSelect getRandomSelect(DataFusionGlobalState state) { public void setFetchColumnsString(String selectExpr) { this.fetchColumnsString = Optional.of(selectExpr); } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (DataFusionExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (DataFusionJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return DataFusionToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java index 88e486f47..d56493fa2 100644 --- a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java @@ -13,22 +13,31 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionDataType; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.DataFusionToStringVisitor; import sqlancer.datafusion.ast.DataFusionBinaryOperation; import sqlancer.datafusion.ast.DataFusionColumnReference; import sqlancer.datafusion.ast.DataFusionExpression; import sqlancer.datafusion.ast.DataFusionFunction; +import sqlancer.datafusion.ast.DataFusionJoin; +import sqlancer.datafusion.ast.DataFusionSelect; +import sqlancer.datafusion.ast.DataFusionTableReference; import sqlancer.datafusion.ast.DataFusionUnaryPostfixOperation; import sqlancer.datafusion.ast.DataFusionUnaryPrefixOperation; import sqlancer.datafusion.gen.DataFusionBaseExpr.ArgumentType; import sqlancer.datafusion.gen.DataFusionBaseExpr.DataFusionBaseExprType; public final class DataFusionExpressionGenerator - extends TypedExpressionGenerator { + extends TypedExpressionGenerator implements + NoRECGenerator { + private List tables; private final DataFusionGlobalState globalState; public DataFusionExpressionGenerator(DataFusionGlobalState globalState) { @@ -100,7 +109,8 @@ protected DataFusionExpression generateExpression(DataFusionDataType type, int d case BINARY: dfAssert(randomExpr.argTypes.size() == 2 && randomExpr.nArgs == 2, "Binrary expression should only have 2 argument" + randomExpr.argTypes); - List argTypeList = new ArrayList<>(); // types of current expression's input arguments + List argTypeList = new ArrayList<>(); // types of current expression's input + // arguments for (ArgumentType argumentType : randomExpr.argTypes) { if (argumentType instanceof ArgumentType.Fixed) { ArgumentType.Fixed possibleArgTypes = (ArgumentType.Fixed) randomExpr.argTypes.get(0); @@ -134,7 +144,8 @@ protected DataFusionExpression generateExpression(DataFusionDataType type, int d public DataFusionExpression generateFunctionExpression(DataFusionDataType type, int depth, DataFusionBaseExpr exprType) { if (exprType.isVariadic || Randomly.getBooleanWithSmallProbability()) { - // TODO(datafusion) maybe add possible types. e.g. some function have signature variadic(INT/DOUBLE), then + // TODO(datafusion) maybe add possible types. e.g. some function have signature + // variadic(INT/DOUBLE), then // only randomly pick from INT and DOUBLE int nArgs = Randomly.smallNumber(); // 0, 2, 4, ... smaller one is more likely return new DataFusionFunction(generateExpressions(nArgs), exprType); @@ -222,4 +233,66 @@ public String getTextRepresentation() { } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + List randomTables = Randomly.nonEmptySubset(tables.getTables()); + int maxSize = Randomly.fromOptions(1, 2, 3, 4); + if (randomTables.size() > maxSize) { + randomTables = randomTables.subList(0, maxSize); + } + this.columns = DataFusionTable.getAllColumns(randomTables); + this.tables = randomTables; + + return this; + } + + @Override + public DataFusionExpression generateBooleanExpression() { + return generateExpression(DataFusionDataType.BOOLEAN); + } + + @Override + public DataFusionSelect generateSelect() { + return new DataFusionSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new DataFusionTableReference(t)) + .collect(Collectors.toList()); + List joins = DataFusionJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new DataFusionTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(DataFusionSelect select, DataFusionExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + select.setFetchColumnsString("COUNT(*)"); + } else { + List allColumns = columns.stream().map((c) -> new DataFusionColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(DataFusionSelect select, DataFusionExpression whereCondition) { + String fetchColumn = String.format("COUNT(CASE WHEN %S THEN 1 ELSE NULL END)", + DataFusionToStringVisitor.asString(whereCondition)); + select.setFetchColumnsString(fetchColumn); + select.setWhereClause(null); + + return select.asString(); + } } From dded5b9bca73322634b33147c9a9a3c4bcbb805b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 3 Sep 2024 15:00:05 +0800 Subject: [PATCH 514/856] [DataFusion] Use common NoREC oracle --- src/sqlancer/datafusion/DataFusionErrors.java | 12 ++- .../test/DataFusionNoRECOracle.java | 99 +++++-------------- 2 files changed, 35 insertions(+), 76 deletions(-) diff --git a/src/sqlancer/datafusion/DataFusionErrors.java b/src/sqlancer/datafusion/DataFusionErrors.java index d4b208f5b..4be35017e 100644 --- a/src/sqlancer/datafusion/DataFusionErrors.java +++ b/src/sqlancer/datafusion/DataFusionErrors.java @@ -2,6 +2,9 @@ import static sqlancer.datafusion.DataFusionUtil.dfAssert; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.query.ExpectedErrors; public final class DataFusionErrors { @@ -17,7 +20,8 @@ private DataFusionErrors() { * Note now it's implemented this way for simplicity This way might cause false negative, because Q1 and Q2 should * both succeed or both fail TODO(datafusion): ensure both succeed or both fail */ - public static void registerExpectedExecutionErrors(ExpectedErrors errors) { + public static List getExpectedExecutionErrors() { + ArrayList errors = new ArrayList<>(); /* * Expected */ @@ -40,5 +44,11 @@ public static void registerExpectedExecutionErrors(ExpectedErrors errors) { errors.add("Physical plan does not support logical expression AggregateFunction"); // False positive: when aggr // is generated in where // clause + + return errors; + } + + public static void registerExpectedExecutionErrors(ExpectedErrors errors) { + errors.addAll(getExpectedExecutionErrors()); } } diff --git a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java index 55f18fecd..36ab97854 100644 --- a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java +++ b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java @@ -1,95 +1,44 @@ package sqlancer.datafusion.test; -import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.ERROR; -import static sqlancer.datafusion.ast.DataFusionSelect.getRandomSelect; - import java.sql.SQLException; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.datafusion.DataFusionErrors; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionToStringVisitor; -import sqlancer.datafusion.DataFusionUtil; +import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionJoin; import sqlancer.datafusion.ast.DataFusionSelect; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; -public class DataFusionNoRECOracle extends NoRECBase - implements TestOracle { +public class DataFusionNoRECOracle implements TestOracle { - private final DataFusionGlobalState state; + NoRECOracle oracle; public DataFusionNoRECOracle(DataFusionGlobalState globalState) { - super(globalState); - this.state = globalState; - DataFusionErrors.registerExpectedExecutionErrors(errors); + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } - /* - * Non-Optimizing Reference Engine Construction q1: SELECT [expr1] FROM [expr2] WHERE [expr3] q2: SELECT [expr3] - * FROM [expr2] - * - * Oracle Check: q1's result size equals to `true` count in q2's result set - */ @Override public void check() throws SQLException { - /* - * Setup Q1 and Q2 - */ - // generate a random: - // SELECT [expr1] FROM [expr2] WHERE [expr3] - DataFusionSelect randomSelect = getRandomSelect(state); - // Q1: SELECT count(*) FROM [expr2] WHERE [expr3] - DataFusionSelect q1 = new DataFusionSelect(); - q1.setFetchColumnsString("COUNT(*)"); - q1.setFromList(randomSelect.getFromList()); - q1.setWhereClause(randomSelect.getWhereClause()); - // Q2: SELECT count(case when [expr3] then 1 else null end) FROM [expr2] - DataFusionSelect q2 = new DataFusionSelect(); - String selectExpr = String.format("COUNT(CASE WHEN %S THEN 1 ELSE NULL END)", - DataFusionToStringVisitor.asString(randomSelect.getWhereClause())); - q2.setFetchColumnsString(selectExpr); - q2.setFromList(randomSelect.getFromList()); - q2.setWhereClause(null); - - /* - * Execute Q1 and Q2 - */ - String q1String = DataFusionToStringVisitor.asString(q1); - String q2String = DataFusionToStringVisitor.asString(q2); - List q1ResultSet = null; - List q2ResultSet = null; - try { - q1ResultSet = ComparatorHelper.getResultSetFirstColumnAsString(q1String, errors, state); - q2ResultSet = ComparatorHelper.getResultSetFirstColumnAsString(q2String, errors, state); - } catch (AssertionError e) { - // Append detailed error message - String replay = DataFusionUtil.getReplay(state.getDatabaseName()); - String newMessage = e.getMessage() + "\n" + e.getCause() + "\n" + replay + "\n"; - state.dfLogger.appendToLog(ERROR, newMessage); - - throw new AssertionError(newMessage); - } - - /* - * NoREC check - */ - int count1 = q1ResultSet != null ? Integer.parseInt(q1ResultSet.get(0)) : -1; - int count2 = q2ResultSet != null ? Integer.parseInt(q2ResultSet.get(0)) : -1; - if (count1 != count2) { - StringBuilder errorMessage = new StringBuilder().append("NoREC oracle violated:\n") - .append(" Q1(result size ").append(count1).append("):").append(q1String).append(";\n") - .append(" Q2(result size ").append(count2).append("):").append(q2String).append(";\n") - .append("=======================================\n").append("Reproducer: \n"); - - String replay = DataFusionUtil.getReplay(state.getDatabaseName()); + oracle.check(); + } - String errorLog = errorMessage.toString() + replay + "\n"; - String indentedErrorLog = errorLog.replaceAll("(?m)^", " "); - state.dfLogger.appendToLog(ERROR, errorLog); + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } - throw new AssertionError("\n\n" + indentedErrorLog); - } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From 97095e89b8a4f44c894744eb12036fe7d2266102 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 4 Sep 2024 18:29:39 +0800 Subject: [PATCH 515/856] [DataFusion] Update tests --- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestDataFusion.java | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 1845aa422..9a3b4c63d 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -6,6 +6,7 @@ public class TestConfig { public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; + public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestDataFusion.java b/test/sqlancer/dbms/TestDataFusion.java index ae568e40e..b2b5e2a1a 100644 --- a/test/sqlancer/dbms/TestDataFusion.java +++ b/test/sqlancer/dbms/TestDataFusion.java @@ -10,9 +10,7 @@ public class TestDataFusion { @Test public void testDataFusion() { - String datafusionAvailable = System.getenv("DATAFUSION_AVAILABLE"); - boolean datafusionIsAvailable = datafusionAvailable != null && datafusionAvailable.equalsIgnoreCase("true"); - assumeTrue(datafusionIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.DATAFUSION_ENV)); assertEquals(0, Main.executeMain("--random-seed", "0", "--num-threads", "1", // TODO(datafusion) update when // multithread is supported From 22d4bb48da62a1e4443348a925f4e53ce2807431 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 30 Aug 2024 12:53:53 +0800 Subject: [PATCH 516/856] [ClickHouse] Implement NoREC expression generator interface --- .../clickhouse/ClickHouseToStringVisitor.java | 6 +- .../clickhouse/ast/ClickHouseExpression.java | 14 ++- .../clickhouse/ast/ClickHouseSelect.java | 45 +++++++-- .../gen/ClickHouseExpressionGenerator.java | 91 ++++++++++++++++++- 4 files changed, 142 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java index cebce8fc2..29bcddcb7 100644 --- a/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java +++ b/src/sqlancer/clickhouse/ClickHouseToStringVisitor.java @@ -74,10 +74,10 @@ public void visit(ClickHouseSelect select, boolean inner) { } visit(select.getFetchColumns()); - ClickHouseExpression fromClause = select.getFromClause(); - if (fromClause != null) { + List fromList = select.getFromList(); + if (fromList != null) { sb.append(" FROM "); - visit(fromClause); + visit(fromList); } List joins = select.getJoinClauses(); if (!joins.isEmpty()) { diff --git a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java index f5cd2ed1e..eff88d012 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseExpression.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseExpression.java @@ -1,9 +1,13 @@ package sqlancer.clickhouse.ast; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; import sqlancer.common.visitor.BinaryOperation; import sqlancer.common.visitor.UnaryOperation; -public abstract class ClickHouseExpression { +public abstract class ClickHouseExpression implements Expression { public ClickHouseConstant getExpectedValue() { return null; @@ -61,7 +65,8 @@ public String getOperatorRepresentation() { } } - public static class ClickHouseJoin extends ClickHouseExpression { + public static class ClickHouseJoin extends ClickHouseExpression + implements Join { // TODO: support ANY, ALL, ASOF modifiers // LEFT_SEMI, RIGHT_SEMI are not deterministic as ClickHouse allows to read columns from // whitelist table as well @@ -71,7 +76,7 @@ public enum JoinType { private final ClickHouseTableReference leftTable; private final ClickHouseTableReference rightTable; - private ClickHouseJoinOnClause onClause; + private ClickHouseExpression onClause; private final ClickHouseJoin.JoinType type; public ClickHouseJoin(ClickHouseTableReference leftTable, ClickHouseTableReference rightTable, @@ -109,7 +114,8 @@ public ClickHouseJoin.JoinType getType() { return type; } - public void setOnClause(ClickHouseJoinOnClause onClause) { + @Override + public void setOnClause(ClickHouseExpression onClause) { this.onClause = onClause; } diff --git a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java index 3904b767b..61aeffa4d 100644 --- a/src/sqlancer/clickhouse/ast/ClickHouseSelect.java +++ b/src/sqlancer/clickhouse/ast/ClickHouseSelect.java @@ -3,10 +3,16 @@ import java.util.Collections; import java.util.List; -public class ClickHouseSelect extends ClickHouseExpression { +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ClickHouseToStringVisitor; +import sqlancer.common.ast.newast.Select; + +public class ClickHouseSelect extends ClickHouseExpression implements + Select { private ClickHouseSelect.SelectType fromOptions = ClickHouseSelect.SelectType.ALL; - private ClickHouseExpression fromClause; + private List fromClauses; private ClickHouseExpression whereClause; private List groupByClause = Collections.emptyList(); private ClickHouseExpression limitClause; @@ -24,12 +30,13 @@ public void setSelectType(ClickHouseSelect.SelectType fromOptions) { this.setFromOptions(fromOptions); } - public void setFromClause(ClickHouseExpression fromClause) { - this.fromClause = fromClause; + public void setFromClause(ClickHouseExpression fromList) { + this.fromClauses = List.of(fromList); } - public ClickHouseExpression getFromClause() { - return fromClause; + @Override + public List getFromList() { + return fromClauses; } public ClickHouseSelect.SelectType getFromOptions() { @@ -40,68 +47,94 @@ public void setFromOptions(ClickHouseSelect.SelectType fromOptions) { this.fromOptions = fromOptions; } + @Override public ClickHouseExpression getWhereClause() { return whereClause; } + @Override public void setWhereClause(ClickHouseExpression whereClause) { this.whereClause = whereClause; } + @Override public void setGroupByClause(List groupByClause) { this.groupByClause = groupByClause; } + @Override public List getGroupByClause() { return groupByClause; } + @Override public void setLimitClause(ClickHouseExpression limitClause) { this.limitClause = limitClause; } + @Override public ClickHouseExpression getLimitClause() { return limitClause; } + @Override public List getOrderByClauses() { return orderByClause; } + @Override public void setOrderByClauses(List orderBy) { this.orderByClause = orderBy; } + @Override public void setOffsetClause(ClickHouseExpression offsetClause) { this.offsetClause = offsetClause; } + @Override public ClickHouseExpression getOffsetClause() { return offsetClause; } + @Override public void setFetchColumns(List fetchColumns) { this.fetchColumns = fetchColumns; } + @Override public List getFetchColumns() { return fetchColumns; } + @Override public void setJoinClauses(List joinStatements) { this.joinStatements = joinStatements; } + @Override public List getJoinClauses() { return joinStatements; } + @Override public void setHavingClause(ClickHouseExpression havingClause) { this.havingClause = havingClause; } + @Override public ClickHouseExpression getHavingClause() { assert orderByClause != null; return havingClause; } + + @Override + public String asString() { + return ClickHouseToStringVisitor.asString(this); + } + + @Override + public void setFromList(List fromList) { + this.fromClauses = fromList; + } } diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index 03750fee0..b523703a3 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -11,13 +11,18 @@ import sqlancer.clickhouse.ClickHouseSchema; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseLancerDataType; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ast.ClickHouseAggregate; +import sqlancer.clickhouse.ast.ClickHouseAggregate.ClickHouseAggregateFunction; +import sqlancer.clickhouse.ast.ClickHouseAliasOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryArithmeticOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryComparisonOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseBinaryLogicalOperation; import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; +import sqlancer.clickhouse.ast.ClickHouseSelect; import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.ast.ClickHouseUnaryFunctionOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPostfixOperation; @@ -25,14 +30,18 @@ import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation; import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; public class ClickHouseExpressionGenerator - extends TypedExpressionGenerator { + extends TypedExpressionGenerator implements + NoRECGenerator { private final ClickHouseGlobalState globalState; public boolean allowAggregateFunctions; + private List tables; private final List columnRefs; public ClickHouseExpressionGenerator(ClickHouseGlobalState globalState) { @@ -345,4 +354,84 @@ public ClickHouseExpression negatePredicate(ClickHouseExpression predicate) { public ClickHouseExpression isNull(ClickHouseExpression expr) { return new ClickHouseUnaryPostfixOperation(expr, ClickHouseUnaryPostfixOperator.IS_NULL, false); } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.tables = tables.getTables(); + this.columns = tables.getColumns(); + return this; + } + + @Override + public ClickHouseExpression generateBooleanExpression() { + List columnRefs = columns.stream() + .map(c -> c.asColumnReference(c.getTable().getName())).collect(Collectors.toList()); + return generateExpressionWithColumns(columnRefs, 5); + } + + @Override + public ClickHouseSelect generateSelect() { + return new ClickHouseSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinStatements = new ArrayList<>(); + if (globalState.getClickHouseOptions().testJoins && Randomly.getBoolean()) { + return joinStatements; + } + List leftTables = new ArrayList<>(); + leftTables.add(new ClickHouseTableReference(tables.get(0), null)); + if (Randomly.getBoolean() && !tables.isEmpty()) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + for (int i = 0; i < nrJoinClauses; i++) { + ClickHouseTableReference leftTable = leftTables + .get((int) Randomly.getNotCachedInteger(0, leftTables.size() - 1)); + ClickHouseTableReference rightTable = new ClickHouseTableReference(Randomly.fromList(tables), + "right_" + i); + ClickHouseExpression.ClickHouseJoinOnClause joinClause = generateJoinClause(leftTable, rightTable); + ClickHouseExpression.ClickHouseJoin.JoinType options = Randomly + .fromOptions(ClickHouseExpression.ClickHouseJoin.JoinType.values()); + ClickHouseExpression.ClickHouseJoin j = new ClickHouseExpression.ClickHouseJoin(leftTable, rightTable, + options, joinClause); + joinStatements.add(j); + leftTables.add(rightTable); + } + } + return joinStatements; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new ClickHouseTableReference(t, null)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(ClickHouseSelect select, ClickHouseExpression whereCondition, + boolean shouldUseAggregate) { + List filteredColumns = Randomly.extractNrRandomColumns(columns, + (int) Randomly.getNotCachedInteger(1, columns.size())); + if (shouldUseAggregate) { + ClickHouseAggregate aggr = new ClickHouseAggregate( + new ClickHouseColumnReference(ClickHouseColumn.createDummy("*", null), null, null), + ClickHouseAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + select.setFetchColumns(filteredColumns.stream().map(c -> c.asColumnReference(c.getTable().getName())) + .collect(Collectors.toList())); + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(ClickHouseSelect select, ClickHouseExpression whereCondition) { + ClickHouseExpression inner = new ClickHouseAliasOperation(whereCondition, "check"); + + select.setFetchColumns(List.of(inner)); + select.setWhereClause(null); + return "SELECT SUM(check <> 0) FROM (" + select.asString() + ") as res"; + } } From 2c4d09a70e98ddae5cc692add54c6965ef0d0287 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 30 Aug 2024 12:54:28 +0800 Subject: [PATCH 517/856] [ClickHouse] Use common NoREC oracle --- .../oracle/norec/ClickHouseNoRECOracle.java | 119 +++--------------- 1 file changed, 18 insertions(+), 101 deletions(-) diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java index 9818d198b..d2acc24d6 100644 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java @@ -1,128 +1,45 @@ package sqlancer.clickhouse.oracle.norec; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.clickhouse.ClickHouseErrors; import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ClickHouseToStringVisitor; -import sqlancer.clickhouse.ast.ClickHouseAliasOperation; -import sqlancer.clickhouse.ast.ClickHouseColumnReference; import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.ast.ClickHouseTableReference; import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; -public class ClickHouseNoRECOracle extends NoRECBase - implements TestOracle { +public class ClickHouseNoRECOracle implements TestOracle { - private final ClickHouseSchema schema; + NoRECOracle oracle; public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { - super(globalState); - this.schema = globalState.getSchema(); - ClickHouseErrors.addExpectedExpressionErrors(errors); + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state); - List tables = schema.getRandomTableNonEmptyTables().getTables(); - ClickHouseTableReference table = new ClickHouseTableReference( - tables.get((int) Randomly.getNotCachedInteger(0, tables.size() - 1)), "left"); - List columns = table.getColumnReferences(); - - List joinStatements = new ArrayList<>(); - if (state.getClickHouseOptions().testJoins && Randomly.getBoolean()) { - joinStatements = gen.getRandomJoinClauses(table, tables); - columns.addAll(joinStatements.stream().flatMap(j -> j.getRightTable().getColumnReferences().stream()) - .collect(Collectors.toList())); - } - gen.addColumns(columns); - - ClickHouseExpression randomWhereCondition = gen.generateExpressionWithColumns(columns, 5); - int secondCount = getSecondQuery(table, randomWhereCondition, joinStatements); - int firstCount = getFirstQueryCount(table, columns, randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getSecondQuery(ClickHouseExpression table, ClickHouseExpression whereClause, - List joins) throws SQLException { - ClickHouseSelect select = new ClickHouseSelect(); - - ClickHouseExpression inner = new ClickHouseAliasOperation(whereClause, "check"); - - select.setFetchColumns(Arrays.asList(inner)); - select.setFromClause(table); - select.setJoinClauses(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(check <> 0) FROM (" + ClickHouseToStringVisitor.asString(select) - + ") as res"; - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getFirstQueryCount(ClickHouseExpression tableList, List columns, - ClickHouseExpression randomWhereCondition, List joins) - throws SQLException { - ClickHouseSelect select = new ClickHouseSelect(); - List filteredColumns = Randomly.extractNrRandomColumns(columns, - (int) Randomly.getNotCachedInteger(1, columns.size())); - select.setFetchColumns( - filteredColumns.stream().map(c -> (ClickHouseExpression) c).collect(Collectors.toList())); - select.setFromClause(tableList); - select.setWhereClause(randomWhereCondition); - select.setJoinClauses(joins); - int firstCount = 0; - optimizedQueryString = ClickHouseToStringVisitor.asString(select); - SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - SQLancerResultSet rs; - - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(optimizedQueryString, e); - } - if (rs == null) { - return -1; - } - - while (rs.next()) { - firstCount++; - } - rs.close(); - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From f43858d50654065b17d4951d68719d1a0d2f5c06 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 30 Aug 2024 13:48:19 +0800 Subject: [PATCH 518/856] [ClickHouse] Refactor environment check in tests --- test/sqlancer/dbms/TestClickHouse.java | 56 +++++++------------------- test/sqlancer/dbms/TestConfig.java | 1 + 2 files changed, 15 insertions(+), 42 deletions(-) diff --git a/test/sqlancer/dbms/TestClickHouse.java b/test/sqlancer/dbms/TestClickHouse.java index ae861dfd9..87d7713fe 100644 --- a/test/sqlancer/dbms/TestClickHouse.java +++ b/test/sqlancer/dbms/TestClickHouse.java @@ -11,9 +11,7 @@ public class TestClickHouse { @Test public void testClickHouseTLPWhereGroupBy() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "--database-prefix", "T1_", "clickhouse", @@ -22,9 +20,7 @@ public void testClickHouseTLPWhereGroupBy() { @Test public void testClickHouseTLPWhere() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "--database-prefix", "T2_", "clickhouse", @@ -33,9 +29,7 @@ public void testClickHouseTLPWhere() { @Test public void testClickHouseTLPHaving() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", @@ -44,9 +38,7 @@ public void testClickHouseTLPHaving() { @Test public void testClickHouseTLPGroupBy() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -55,9 +47,7 @@ public void testClickHouseTLPGroupBy() { @Test public void testClickHouseTLPDistinct() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -66,9 +56,7 @@ public void testClickHouseTLPDistinct() { @Test public void testClickHouseTLPAggregate() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -77,9 +65,7 @@ public void testClickHouseTLPAggregate() { @Test public void testClickHouseNoREC() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", @@ -88,9 +74,7 @@ public void testClickHouseNoREC() { @Test public void testClickHouseTLPWhereGroupByWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "--database-prefix", "T8_", "clickhouse", @@ -99,9 +83,7 @@ public void testClickHouseTLPWhereGroupByWithJoins() { @Test public void testClickHouseTLPWhereWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", "--password", "", "--database-prefix", "T9_", "clickhouse", @@ -110,9 +92,7 @@ public void testClickHouseTLPWhereWithJoins() { @Test public void testClickHouseTLPHavingWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", @@ -121,9 +101,7 @@ public void testClickHouseTLPHavingWithJoins() { @Test public void testClickHouseTLPGroupByWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -132,9 +110,7 @@ public void testClickHouseTLPGroupByWithJoins() { @Test public void testClickHouseTLPDistinctWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -143,9 +119,7 @@ public void testClickHouseTLPDistinctWithJoins() { @Test public void testClickHouseTLPAggregateWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "5", "--username", "default", @@ -154,9 +128,7 @@ public void testClickHouseTLPAggregateWithJoins() { @Test public void testClickHouseNoRECWithJoins() { - String clickHouseAvailable = System.getenv("CLICKHOUSE_AVAILABLE"); - boolean clickHouseIsAvailable = clickHouseAvailable != null && clickHouseAvailable.equalsIgnoreCase("true"); - assumeTrue(clickHouseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CLICKHOUSE_ENV)); assertEquals(0, Main.executeMain("--log-each-select", "true", "--print-failed", "false", "--timeout-seconds", "60", "--num-queries", TestConfig.NUM_QUERIES, "--num-threads", "1", "--username", "default", diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index d70c0ae2f..2f2c18e83 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -4,6 +4,7 @@ public class TestConfig { public static final String NUM_QUERIES = "1000"; public static final String SECONDS = "300"; + public static final String CLICKHOUSE_ENV = "CLICKHOUSE_AVAILABLE"; public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; From bfb9b237fd1dc5fa0fed7bc5d9019c20d604d4fa Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 9 Sep 2024 13:49:01 +0800 Subject: [PATCH 519/856] [HSQLDB] Add CI & tests --- .github/workflows/main.yml | 20 ++++++++++++++++++++ test/sqlancer/dbms/TestHSQLDBNoREC.java | 15 +++++++++++++++ test/sqlancer/dbms/TestHSQLDBTLP.java | 15 +++++++++++++++ 3 files changed, 50 insertions(+) create mode 100644 test/sqlancer/dbms/TestHSQLDBNoREC.java create mode 100644 test/sqlancer/dbms/TestHSQLDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68b02c443..dfa1b1238 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -268,6 +268,26 @@ jobs: - name: Run Tests run: H2_AVAILABLE=true mvn -Dtest=TestH2 test + hsqldb: + name: DBMS Tests (HSQLB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + mvn -Dtest=TestHSQLDBNoREC test + mvn -Dtest=TestHSQLDBTLP test + mariadb: name: DBMS Tests (MariaDB) runs-on: ubuntu-latest diff --git a/test/sqlancer/dbms/TestHSQLDBNoREC.java b/test/sqlancer/dbms/TestHSQLDBNoREC.java new file mode 100644 index 000000000..721fbb126 --- /dev/null +++ b/test/sqlancer/dbms/TestHSQLDBNoREC.java @@ -0,0 +1,15 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestHSQLDBNoREC { + @Test + public void testHSQLDBNoREC() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "hsqldb", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestHSQLDBTLP.java b/test/sqlancer/dbms/TestHSQLDBTLP.java new file mode 100644 index 000000000..a0b9c18d9 --- /dev/null +++ b/test/sqlancer/dbms/TestHSQLDBTLP.java @@ -0,0 +1,15 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestHSQLDBTLP { + @Test + public void testHSQLDBTLP() { + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "hsqldb", "--oracle", "WHERE" })); + } +} From 40c9c113f2c245479700e41a36afb487199b52a7 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 9 Sep 2024 14:47:38 +0800 Subject: [PATCH 520/856] [OceanBase] Add CI & update expected errors --- .github/workflows/main.yml | 25 +++++++++++++++++++ src/sqlancer/oceanbase/OceanBaseErrors.java | 1 + .../oceanbase/gen/OceanBaseAlterTable.java | 2 +- .../gen/OceanBaseDeleteGenerator.java | 2 +- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestOceanBaseNoREC.java | 5 +--- test/sqlancer/dbms/TestOceanBasePQS.java | 5 +--- test/sqlancer/dbms/TestOceanBaseTLP.java | 5 +--- 8 files changed, 32 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aaafeb8cd..f38173d96 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -384,6 +384,31 @@ jobs: MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP + oceanbase: + name: DBMS Tests (OceanBase) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up OceanBase + run: | + docker run -p 2881:2881 --name oceanbase-ce -e MODE=mini -d oceanbase/oceanbase-ce:4.2.1-lts + sleep 120 + mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" + - name: Run Tests + run: | + OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseNoREC + OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBasePQS + OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseTLP postgres: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index 822c5497d..5ad9dd3f3 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -20,6 +20,7 @@ public static List getExpressionErrors() { errors.add("Truncated incorrect DOUBLE value"); errors.add("Invalid numeric"); errors.add("Data truncated for argument"); + errors.add("Data truncated for column"); return errors; } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java index e67256448..7e1a69dfc 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseAlterTable.java @@ -56,7 +56,7 @@ private SQLQueryAdapter create() { case COMPRESSION: sb.append("COMPRESSION "); sb.append("'"); - sb.append(Randomly.fromOptions("ZLIB_1.0", "LZ4_1.0", "NONE")); + sb.append(Randomly.fromOptions("LZ4_1.0", "NONE")); sb.append("'"); break; default: diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index f2a6b2731..ea1cb36e8 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -44,7 +44,7 @@ private SQLQueryAdapter generate() { errors.addAll(Arrays.asList("doesn't have this option", "Truncated incorrect DOUBLE value", "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index", "Incorrect value", "Out of range value for column", - "Data truncation: %s value is out of range in '%s'")); + "Data truncation:")); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 2f2c18e83..b51309dfe 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -11,6 +11,7 @@ public class TestConfig { public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; + public static final String OCEANBASE_ENV = "OCEANBASE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestOceanBaseNoREC.java b/test/sqlancer/dbms/TestOceanBaseNoREC.java index 9c650265c..c687590de 100644 --- a/test/sqlancer/dbms/TestOceanBaseNoREC.java +++ b/test/sqlancer/dbms/TestOceanBaseNoREC.java @@ -9,12 +9,9 @@ public class TestOceanBaseNoREC { - String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); - boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); - @Test public void testNoREC() { - assumeTrue(oceanBaseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.OCEANBASE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--database-prefix", "norecdb", "--num-queries", TestConfig.NUM_QUERIES, diff --git a/test/sqlancer/dbms/TestOceanBasePQS.java b/test/sqlancer/dbms/TestOceanBasePQS.java index adb2007d5..404977398 100644 --- a/test/sqlancer/dbms/TestOceanBasePQS.java +++ b/test/sqlancer/dbms/TestOceanBasePQS.java @@ -9,12 +9,9 @@ public class TestOceanBasePQS { - String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); - boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); - @Test public void testPQS() { - assumeTrue(oceanBaseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.OCEANBASE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC_SPECIALCHAR", diff --git a/test/sqlancer/dbms/TestOceanBaseTLP.java b/test/sqlancer/dbms/TestOceanBaseTLP.java index fe84118e6..cf86c2078 100644 --- a/test/sqlancer/dbms/TestOceanBaseTLP.java +++ b/test/sqlancer/dbms/TestOceanBaseTLP.java @@ -9,12 +9,9 @@ public class TestOceanBaseTLP { - String oceanBaseAvailable = System.getenv("OCEANBASE_AVAILABLE"); - boolean oceanBaseIsAvailable = oceanBaseAvailable != null && oceanBaseAvailable.equalsIgnoreCase("true"); - @Test public void testTLP() { - assumeTrue(oceanBaseIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.OCEANBASE_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--database-prefix", "tlpdb", "--num-queries", TestConfig.NUM_QUERIES, From b13132f5d4d3a6f4c1b3d04d6db6ff157f0db288 Mon Sep 17 00:00:00 2001 From: Marcus Gartner Date: Mon, 9 Sep 2024 13:09:07 -0400 Subject: [PATCH 521/856] Update CockroachDB to v24.2.0 --- .github/workflows/main.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9773919fa..459db4425 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -138,11 +138,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.1.0-rc.2.linux-amd64.tgz | tar xvz - cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: | COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBNoREC test @@ -165,11 +165,11 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.1.0-rc.2.linux-amd64.tgz | tar xvz - cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach start-single-node --insecure & + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & sleep 10 - name: Create SQLancer user - run: cd cockroach-v24.1.0-rc.2.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. + run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test From 47e7ab73aa36ec2c957668dd8300b3528c685bba Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 10 Sep 2024 15:58:26 +0800 Subject: [PATCH 522/856] [OceanBase] Implement expression generator interfaces --- .../oceanbase/ast/OceanBaseExpression.java | 5 +- src/sqlancer/oceanbase/ast/OceanBaseJoin.java | 9 +- .../oceanbase/ast/OceanBaseSelect.java | 21 ++- .../gen/OceanBaseExpressionGenerator.java | 153 +++++++++++++++++- 4 files changed, 184 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/oceanbase/ast/OceanBaseExpression.java b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java index 195a05967..b721d711f 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseExpression.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseExpression.java @@ -1,6 +1,9 @@ package sqlancer.oceanbase.ast; -public interface OceanBaseExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; + +public interface OceanBaseExpression extends Expression { default OceanBaseConstant getExpectedValue() { throw new AssertionError("PQS not supported for this operator"); diff --git a/src/sqlancer/oceanbase/ast/OceanBaseJoin.java b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java index 9e8271bbf..4855c0808 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseJoin.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseJoin.java @@ -1,10 +1,17 @@ package sqlancer.oceanbase.ast; -public class OceanBaseJoin implements OceanBaseExpression { +import sqlancer.common.ast.newast.Join; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; + +public class OceanBaseJoin implements OceanBaseExpression, Join { @Override public OceanBaseConstant getExpectedValue() { throw new UnsupportedOperationException(); } + @Override + public void setOnClause(OceanBaseExpression onClause) { + } } diff --git a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java index a673492c2..c4bbc1b87 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseSelect.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseSelect.java @@ -5,8 +5,13 @@ import java.util.List; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.OceanBaseVisitor; -public class OceanBaseSelect extends SelectBase implements OceanBaseExpression { +public class OceanBaseSelect extends SelectBase + implements OceanBaseExpression, Select { private SelectType fromOptions = SelectType.ALL; private List modifiers = Collections.emptyList(); @@ -60,4 +65,18 @@ public OceanBaseStringExpression getHint() { return hint; } + @Override + public void setJoinClauses(List joinStatements) { + } + + @Override + public List getJoinClauses() { + return List.of(); + } + + @Override + public String asString() { + return OceanBaseVisitor.asString(this); + } + } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java index f4f573698..3c17e5084 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java @@ -3,13 +3,20 @@ import java.sql.Connection; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseDataType; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseRowValue; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseAggregate; +import sqlancer.oceanbase.ast.OceanBaseAggregate.OceanBaseAggregateFunction; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation; import sqlancer.oceanbase.ast.OceanBaseBinaryComparisonOperation.BinaryComparisonOperator; import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; @@ -23,15 +30,22 @@ import sqlancer.oceanbase.ast.OceanBaseExists; import sqlancer.oceanbase.ast.OceanBaseExpression; import sqlancer.oceanbase.ast.OceanBaseInOperation; +import sqlancer.oceanbase.ast.OceanBaseJoin; +import sqlancer.oceanbase.ast.OceanBaseSelect; import sqlancer.oceanbase.ast.OceanBaseStringExpression; +import sqlancer.oceanbase.ast.OceanBaseTableReference; +import sqlancer.oceanbase.ast.OceanBaseText; import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; -public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator { +public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator + implements + NoRECGenerator { private OceanBaseGlobalState state; private OceanBaseRowValue rowVal; + private List tables; public OceanBaseExpressionGenerator(OceanBaseGlobalState state) { this.state = state; @@ -209,4 +223,141 @@ public OceanBaseExpression isNull(OceanBaseExpression expr) { return new OceanBaseUnaryPostfixOperation(expr, OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public OceanBaseExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public OceanBaseSelect generateSelect() { + return new OceanBaseSelect(); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(OceanBaseSelect select, OceanBaseExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + OceanBaseExpression aggr = new OceanBaseAggregate( + new OceanBaseColumnReference(new OceanBaseColumn("*", OceanBaseDataType.INT, false, 0, false), + null), + OceanBaseAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + } else { + List allColumns = columns.stream().map((c) -> new OceanBaseColumnReference(c, null)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(OceanBaseSelect select, OceanBaseExpression whereCondition) { + OceanBaseExpression expr = getTrueExpr(whereCondition); + + OceanBaseText asText = new OceanBaseText(expr, " as count", false); + select.setFetchColumns(List.of(asText)); + select.setSelectType(OceanBaseSelect.SelectType.ALL); + + return "SELECT SUM(count) FROM (" + select.asString() + ") as asdf"; + } + + private enum Option { + TRUE, FALSE_NULL, NOT_NOT_TRUE, NOT_FALSE_NOT_NULL, IF, IFNULL, COALESCE + }; + + private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition) { + // we can treat "is true" as combinations of "is flase" and "not","is not true" and "not",etc. + OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE, false); + + OceanBaseUnaryPostfixOperation isFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, false); + + OceanBaseUnaryPostfixOperation isNotFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, true); + + OceanBaseUnaryPostfixOperation isNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); + + OceanBaseUnaryPostfixOperation isNotNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, + OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, true); + + OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); + Option a = Randomly.fromOptions(Option.values()); + switch (a) { + case TRUE: + expr = isTrue; + break; + case FALSE_NULL: + // not((is false) or (is null)) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseBinaryLogicalOperation(isFalse, isNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_NOT_TRUE: + // not(not(is true))) + expr = new OceanBaseUnaryPrefixOperation( + new OceanBaseUnaryPrefixOperation(isTrue, + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), + OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); + break; + case NOT_FALSE_NOT_NULL: + // (is not false) and (is not null) + expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, + OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); + break; + case IF: + // if(1, xx is true, 0) + OceanBaseExpression[] args = new OceanBaseExpression[3]; + args[0] = OceanBaseConstant.createIntConstant(1); + args[1] = isTrue; + args[2] = OceanBaseConstant.createIntConstant(0); + expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); + break; + case IFNULL: + // ifnull(null, xx is true) + OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; + ifArgs[0] = OceanBaseConstant.createNullConstant(); + ifArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); + break; + case COALESCE: + // coalesce(null, xx is true) + OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; + coalesceArgs[0] = OceanBaseConstant.createNullConstant(); + coalesceArgs[1] = isTrue; + expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); + break; + default: + expr = isTrue; + break; + } + return expr; + } } From ca6cb967e5da984645df3b4503e25c1b2d806300 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 10 Sep 2024 15:58:50 +0800 Subject: [PATCH 523/856] [OceanBase] Use common NoREC oracle --- src/sqlancer/oceanbase/OceanBaseErrors.java | 8 + .../oracle/OceanBaseNoRECOracle.java | 225 ++---------------- 2 files changed, 33 insertions(+), 200 deletions(-) diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index 5ad9dd3f3..c539ab129 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; import sqlancer.common.query.ExpectedErrors; @@ -25,6 +26,13 @@ public static List getExpressionErrors() { return errors; } + public static List getExpressionErrorsRegex() { + ArrayList errors = new ArrayList<>(); + errors.add(Pattern.compile("Unknown column '.+' in 'order clause'")); + + return errors; + } + public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); } diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java index cc98dcf0a..0ab0e6d2f 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java @@ -1,224 +1,49 @@ package sqlancer.oceanbase.oracle; import java.sql.SQLException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.Randomly; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.OceanBaseVisitor; -import sqlancer.oceanbase.ast.OceanBaseAggregate; -import sqlancer.oceanbase.ast.OceanBaseBinaryLogicalOperation; -import sqlancer.oceanbase.ast.OceanBaseColumnName; -import sqlancer.oceanbase.ast.OceanBaseComputableFunction; -import sqlancer.oceanbase.ast.OceanBaseComputableFunction.OceanBaseFunction; -import sqlancer.oceanbase.ast.OceanBaseConstant; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseJoin; import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.ast.OceanBaseText; -import sqlancer.oceanbase.ast.OceanBaseUnaryPostfixOperation; -import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation; import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -public class OceanBaseNoRECOracle extends NoRECBase implements TestOracle { +public class OceanBaseNoRECOracle implements TestOracle { - // SELECT COUNT(*) FROM t0 WHERE ; - // SELECT SUM(count) FROM (SELECT IS TRUE as count FROM t0); - // SELECT (SELECT COUNT(*) FROM t0 WHERE c0 IS NOT 0) = (SELECT COUNT(*) FROM - // (SELECT c0 is NOT 0 FROM t0)); - private final OceanBaseSchema s; - private String firstQueryString; - private static final int NOT_FOUND = -1; - - private enum Option { - TRUE, FALSE_NULL, NOT_NOT_TRUE, NOT_FALSE_NOT_NULL, IF, IFNULL, COALESCE - }; + NoRECOracle oracle; public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - errors.add("is out of range"); - // regex - errors.add("unmatched parentheses"); - errors.add("nothing to repeat at offset"); - errors.add("missing )"); - errors.add("missing terminating ]"); - errors.add("range out of order in character class"); - errors.add("unrecognized character after "); - errors.add("Got error '(*VERB) not recognized or malformed"); - errors.add("must be followed by"); - errors.add("malformed number or name after"); - errors.add("digit expected after"); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()) + .with("canceling statement due to statement timeout").with("unmatched parentheses") + .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") + .with("range out of order in character class").with("unrecognized character after ") + .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") + .with("malformed number or name after").with("digit expected after").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - OceanBaseSchema.OceanBaseTable randomTable = s.getRandomTable(); - List columns = randomTable.getColumns(); - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state).setColumns(columns); - OceanBaseExpression randomWhereCondition = gen.generateExpression(); - List groupBys = Collections.emptyList(); // getRandomExpressions(columns); - List tableList = Arrays.asList(randomTable).stream() - .map(t -> new OceanBaseTableReference(t)).collect(Collectors.toList()); - int firstCount = getFirstQueryCount(tableList, randomWhereCondition, groupBys); - int secondCount = getSecondQuery(tableList, randomWhereCondition, groupBys); - if (firstCount != secondCount && firstCount != NOT_FOUND && secondCount != NOT_FOUND) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } + oracle.check(); } - private int getSecondQuery(List tableList, OceanBaseExpression randomWhereCondition, - List groupBys) throws SQLException { - OceanBaseSelect select = new OceanBaseSelect(); - select.setGroupByClause(groupBys); - OceanBaseExpression expr = getTrueExpr(randomWhereCondition); - - OceanBaseText asText = new OceanBaseText(expr, " as count", false); - select.setFetchColumns(Arrays.asList(asText)); - select.setFromList(tableList); - select.setSelectType(OceanBaseSelect.SelectType.ALL); - int secondCount = 0; - - unoptimizedQueryString = "SELECT SUM(count) FROM (" + OceanBaseVisitor.asString(select) + ") as asdf"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(optimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getFirstQueryCount(List tableList, OceanBaseExpression randomWhereCondition, - List groupBys) throws SQLException { - OceanBaseSelect select = new OceanBaseSelect(); - select.setGroupByClause(groupBys); - // SELECT COUNT(t1.c3) FROM t1 WHERE (- (t1.c2)); - // SELECT SUM(count) FROM (SELECT ((- (t1.c2)) IS TRUE) as count FROM t1);; - OceanBaseAggregate aggr = new OceanBaseAggregate(new OceanBaseColumnName( - new OceanBaseSchema.OceanBaseColumn("*", OceanBaseSchema.OceanBaseDataType.INT, false, 0, false)), - OceanBaseAggregate.OceanBaseAggregateFunction.COUNT); - select.setFetchColumns(Arrays.asList(aggr)); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - select.setSelectType(OceanBaseSelect.SelectType.ALL); - int firstCount = 0; - optimizedQueryString = OceanBaseVisitor.asString(select); - SQLQueryAdapter q = new SQLQueryAdapter(optimizedQueryString, errors); - SQLancerResultSet rs; - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try { - rs = q.executeAndGet(state); - } catch (Exception e) { - throw new AssertionError(firstQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - firstCount += rs.getLong(1); - } - rs.close(); - return firstCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition) { - // we can treat "is true" as combinations of "is flase" and "not","is not true" and "not",etc. - OceanBaseUnaryPostfixOperation isTrue = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_TRUE, false); - - OceanBaseUnaryPostfixOperation isFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, false); - - OceanBaseUnaryPostfixOperation isNotFalse = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_FALSE, true); - - OceanBaseUnaryPostfixOperation isNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, false); - - OceanBaseUnaryPostfixOperation isNotNULL = new OceanBaseUnaryPostfixOperation(randomWhereCondition, - OceanBaseUnaryPostfixOperation.UnaryPostfixOperator.IS_NULL, true); - - OceanBaseExpression expr = OceanBaseConstant.createNullConstant(); - Option a = Randomly.fromOptions(Option.values()); - switch (a) { - case TRUE: - expr = isTrue; - break; - case FALSE_NULL: - // not((is false) or (is null)) - expr = new OceanBaseUnaryPrefixOperation( - new OceanBaseBinaryLogicalOperation(isFalse, isNULL, - OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.OR), - OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case NOT_NOT_TRUE: - // not(not(is true))) - expr = new OceanBaseUnaryPrefixOperation( - new OceanBaseUnaryPrefixOperation(isTrue, - OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT), - OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator.NOT); - break; - case NOT_FALSE_NOT_NULL: - // (is not false) and (is not null) - expr = new OceanBaseBinaryLogicalOperation(isNotFalse, isNotNULL, - OceanBaseBinaryLogicalOperation.OceanBaseBinaryLogicalOperator.AND); - break; - case IF: - // if(1, xx is true, 0) - OceanBaseExpression[] args = new OceanBaseExpression[3]; - args[0] = OceanBaseConstant.createIntConstant(1); - args[1] = isTrue; - args[2] = OceanBaseConstant.createIntConstant(0); - expr = new OceanBaseComputableFunction(OceanBaseFunction.IF, args); - break; - case IFNULL: - // ifnull(null, xx is true) - OceanBaseExpression[] ifArgs = new OceanBaseExpression[2]; - ifArgs[0] = OceanBaseConstant.createNullConstant(); - ifArgs[1] = isTrue; - expr = new OceanBaseComputableFunction(OceanBaseFunction.IFNULL, ifArgs); - break; - case COALESCE: - // coalesce(null, xx is true) - OceanBaseExpression[] coalesceArgs = new OceanBaseExpression[2]; - coalesceArgs[0] = OceanBaseConstant.createNullConstant(); - coalesceArgs[1] = isTrue; - expr = new OceanBaseComputableFunction(OceanBaseFunction.COALESCE, coalesceArgs); - break; - default: - expr = isTrue; - break; - } - return expr; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From fbbc86e6019321baaf36f6ebf0adb302c79e5a8b Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 6 Sep 2024 14:46:57 +0800 Subject: [PATCH 524/856] CnosDB] Add CI & tests --- .github/workflows/main.yml | 26 ++++++++++++++++++++ src/sqlancer/cnosdb/CnosDBExpectedError.java | 5 ++++ src/sqlancer/cnosdb/CnosDBSchema.java | 2 +- test/sqlancer/dbms/TestCnosDBNoREC.java | 22 +++++++++++++++++ test/sqlancer/dbms/TestCnosDBTLP.java | 22 +++++++++++++++++ test/sqlancer/dbms/TestConfig.java | 1 + 6 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 test/sqlancer/dbms/TestCnosDBNoREC.java create mode 100644 test/sqlancer/dbms/TestCnosDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aaafeb8cd..552123501 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -92,6 +92,32 @@ jobs: - name: Run Tests run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test + cnosdb: + name: DBMS Tests (CnosDB) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Set up ClickHouse + run: | + docker pull cnosdb/cnosdb:community-latest + docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest + sleep 5 + - name: Run Tests + run: | + CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test + sleep 20 + CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBTLP test + clickhouse: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest diff --git a/src/sqlancer/cnosdb/CnosDBExpectedError.java b/src/sqlancer/cnosdb/CnosDBExpectedError.java index 5932bfdac..61dba101b 100644 --- a/src/sqlancer/cnosdb/CnosDBExpectedError.java +++ b/src/sqlancer/cnosdb/CnosDBExpectedError.java @@ -70,6 +70,11 @@ public static List getExpectedErrors() { errors.add( "Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed."); errors.add("Coercion from"); + + errors.add("Error parsing timestamp"); + errors.add("lpad requested length"); + errors.add("rpad requested length"); + errors.add("No function matches the given name and argument types"); return errors; } diff --git a/src/sqlancer/cnosdb/CnosDBSchema.java b/src/sqlancer/cnosdb/CnosDBSchema.java index 35d9d8887..022969ce5 100644 --- a/src/sqlancer/cnosdb/CnosDBSchema.java +++ b/src/sqlancer/cnosdb/CnosDBSchema.java @@ -38,7 +38,7 @@ public static CnosDBDataType getColumnType(String typeString) { case "bigint unsigned": case "unsigned": return CnosDBDataType.UINT; - case "timestamp": + case "timestamp(nanosecond)": return CnosDBDataType.TIMESTAMP; default: throw new AssertionError(typeString); diff --git a/test/sqlancer/dbms/TestCnosDBNoREC.java b/test/sqlancer/dbms/TestCnosDBNoREC.java new file mode 100644 index 000000000..1a89a972a --- /dev/null +++ b/test/sqlancer/dbms/TestCnosDBNoREC.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCnosDBNoREC { + + @Test + public void testCnosDBNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); + // Run with 0 queries as current implementation is resulting in database crashes + assertEquals(0, + Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", + "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", + "--oracle", "NOREC" })); + } + +} diff --git a/test/sqlancer/dbms/TestCnosDBTLP.java b/test/sqlancer/dbms/TestCnosDBTLP.java new file mode 100644 index 000000000..4b12aa409 --- /dev/null +++ b/test/sqlancer/dbms/TestCnosDBTLP.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCnosDBTLP { + + @Test + public void testCnosDBTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); + // Run with 0 queries as current implementation is resulting in database crashes + assertEquals(0, + Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", + "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", + "--oracle", "QUERY_PARTITIONING" })); + } + +} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 2f2c18e83..a5127900b 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -5,6 +5,7 @@ public class TestConfig { public static final String SECONDS = "300"; public static final String CLICKHOUSE_ENV = "CLICKHOUSE_AVAILABLE"; + public static final String CNOSDB_ENV = "CNOSDB_AVAILABLE"; public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; From fca4b1af286326a4c97ffa9d8e60bf55efe9e0e1 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 6 Sep 2024 13:20:50 +0800 Subject: [PATCH 525/856] [Presto] Add CI & tests --- .github/workflows/main.yml | 26 ++++++++++++++++++ src/sqlancer/presto/PrestoErrors.java | 27 ++++++++++++++----- src/sqlancer/presto/PrestoSchema.java | 4 ++- .../presto/gen/PrestoInsertGenerator.java | 2 +- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestPrestoNoREC.java | 17 ++++++++++++ test/sqlancer/dbms/TestPrestoTLP.java | 19 +++++++++++++ 7 files changed, 88 insertions(+), 8 deletions(-) create mode 100644 test/sqlancer/dbms/TestPrestoNoREC.java create mode 100644 test/sqlancer/dbms/TestPrestoTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6150b8431..a34f31a7b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -457,6 +457,32 @@ jobs: POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresTLP test POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresNoREC test + presto: + name: DBMS Tests (Presto) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Set up Presto + run: | + docker pull prestodb/presto:latest + echo "connector.name=memory" >> memory.properties + docker run -p 8080:8080 -d -v ./memory.properties:/opt/presto-server/etc/catalog/memory.properties --name presto prestodb/presto:latest + sleep 30 + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: | + PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoNoREC test + docker restart presto && sleep 30 + PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoTLP test sqlite: name: DBMS Tests (SQLite) runs-on: ubuntu-latest diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 68a99ef9f..87eba768f 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -13,6 +13,8 @@ private PrestoErrors() { public static List getExpressionErrors() { ArrayList errors = new ArrayList<>(); + errors.addAll(getFunctionErrors()); + // Presto errors errors.add("cannot be applied to"); errors.add("LIKE expression must evaluate to a varchar"); @@ -25,6 +27,7 @@ public static List getExpressionErrors() { // -179769313486231570000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0000) // AS BIGINT)as count FROM t0) as res errors.add("Decimal overflow"); + errors.add("long overflow"); errors.add("multiplication overflow"); errors.add("addition overflow"); errors.add("subtraction overflow"); @@ -39,6 +42,7 @@ public static List getExpressionErrors() { // TODO: check errors.add("io.airlift.slice.Slice cannot be cast to java.lang.Number"); errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); + errors.add("Cannot cast java.lang.String to java.util.List"); errors.add("Unexpected subquery expression in logical plan"); // 9223372036854775808 @@ -51,10 +55,6 @@ public static List getExpressionErrors() { errors.add("Cannot add hour, minutes or seconds to a date"); errors.add("DECIMAL scale must be in range"); - errors.add("multiplication overflow"); - errors.add("addition overflow"); - errors.add("subtraction overflow"); - errors.add("Decimal overflow"); errors.add("IN value and list items must be the same type"); errors.add("is not a valid timestamp literal"); errors.add("Unknown time-zone ID"); @@ -63,6 +63,16 @@ public static List getExpressionErrors() { // ARRAY errors.add("Unknown type: ARRAY"); + // SELECT + errors.add("WHERE clause must evaluate to a boolean"); + errors.add("HAVING clause must evaluate to a boolean"); + errors.add("not yet implemented"); + + errors.add("Value expression and result of subquery must be of the same type for quantified comparison"); + errors.add("All IN list values must be the same type"); + errors.add("All CASE results must be the same type"); + errors.add("Mismatched types"); + return errors; } @@ -107,7 +117,12 @@ private static List getFunctionErrors() { errors.add("Could not choose a best candidate function for the function call"); // monthname errors.add("expected a numeric precision field"); // ROUND errors.add("with non-constant precision is not supported"); // ROUND - + errors.add("Unexpected parameters"); + errors.add("not registered"); + errors.add("Expected: least(E) E:orderable"); + errors.add("Expected: greatest(E) E:orderable"); + errors.add("Expected: max_by(V, K) K:orderable, V, max_by(V, K, bigint) V, K:orderable"); + errors.add("Expected: min_by(V, K) K:orderable, V, min_by(V, K, bigint) V, K:orderable"); return errors; } @@ -116,7 +131,7 @@ public static List getInsertErrors() { ArrayList errors = new ArrayList<>(); errors.addAll(getRegexErrors()); - errors.addAll(getFunctionErrors()); + errors.addAll(getExpressionErrors()); errors.add("NOT NULL constraint failed"); errors.add("PRIMARY KEY or UNIQUE constraint violated"); diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java index 70323a18b..112da7ef2 100644 --- a/src/sqlancer/presto/PrestoSchema.java +++ b/src/sqlancer/presto/PrestoSchema.java @@ -151,8 +151,10 @@ private static PrestoCompositeDataType getColumnType(String typeString) { primitiveType = PrestoDataType.JSON; break; case "ARRAY": + int bracesEnd = typeString.length() - 1; primitiveType = PrestoDataType.ARRAY; - break; + PrestoCompositeDataType elementType = getColumnType(typeString.substring(bracesStart + 1, bracesEnd)); + return new PrestoCompositeDataType(primitiveType, elementType); case "NULL": primitiveType = PrestoDataType.NULL; break; diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index 81e50e1d1..072a22ae0 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -16,7 +16,6 @@ public class PrestoInsertGenerator extends AbstractInsertGenerator { private final PrestoGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public PrestoInsertGenerator(PrestoGlobalState globalState) { this.globalState = globalState; @@ -36,6 +35,7 @@ private SQLQueryAdapter generate() { sb.append(")"); sb.append(" VALUES "); insertColumns(columns); + ExpectedErrors errors = new ExpectedErrors(); PrestoErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, false, false); } diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index b51309dfe..135142694 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -13,6 +13,7 @@ public class TestConfig { public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; public static final String OCEANBASE_ENV = "OCEANBASE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; + public static final String PRESTO_ENV = "PRESTO_AVAILABLE"; public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { diff --git a/test/sqlancer/dbms/TestPrestoNoREC.java b/test/sqlancer/dbms/TestPrestoNoREC.java new file mode 100644 index 000000000..e38e90483 --- /dev/null +++ b/test/sqlancer/dbms/TestPrestoNoREC.java @@ -0,0 +1,17 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPrestoNoREC { + @Test + public void testPrestoNoREC() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.PRESTO_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "presto", "--oracle", "NOREC" })); + } +} diff --git a/test/sqlancer/dbms/TestPrestoTLP.java b/test/sqlancer/dbms/TestPrestoTLP.java new file mode 100644 index 000000000..6ffefc333 --- /dev/null +++ b/test/sqlancer/dbms/TestPrestoTLP.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPrestoTLP { + @Test + public void testPrestoTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.PRESTO_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "--validate-result-size-only", + "true", "--canonicalize-sql-strings", "false", "presto", "--oracle", "QUERY_PARTITIONING" })); + } +} From d9f7cd823435539eaba604aeee2252d5f13e7445 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 9 Sep 2024 15:41:44 +0800 Subject: [PATCH 526/856] [Presto] Check expected error in NoREC result set --- src/sqlancer/presto/test/PrestoNoRECOracle.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java index 606183828..e9919e4d0 100644 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -96,8 +96,16 @@ private int getSecondQuery(List tableList, PrestoExpression ra if (rs == null) { return -1; } - if (rs.next()) { - secondCount += rs.getLong(1); + try { + if (rs.next()) { + secondCount += rs.getLong(1); + } + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw e; + } } rs.close(); return secondCount; From 61b80eaf429957084f92508433a8cafdc566a8cd Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 10 Sep 2024 18:20:26 +0800 Subject: [PATCH 527/856] [Presto] Add bug 23613 --- src/sqlancer/presto/PrestoBugs.java | 11 +++++++++++ src/sqlancer/presto/PrestoErrors.java | 4 ++++ 2 files changed, 15 insertions(+) create mode 100644 src/sqlancer/presto/PrestoBugs.java diff --git a/src/sqlancer/presto/PrestoBugs.java b/src/sqlancer/presto/PrestoBugs.java new file mode 100644 index 000000000..5e347c6f5 --- /dev/null +++ b/src/sqlancer/presto/PrestoBugs.java @@ -0,0 +1,11 @@ +package sqlancer.presto; + +public final class PrestoBugs { + + // https://github.com/prestodb/presto/issues/23613 + public static boolean bug23613 = true; + + private PrestoBugs() { + } + +} diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 87eba768f..4fcb749c3 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -73,6 +73,10 @@ public static List getExpressionErrors() { errors.add("All CASE results must be the same type"); errors.add("Mismatched types"); + if (PrestoBugs.bug23613) { + errors.add("at index 1"); + } + return errors; } From e087048f53ab749ac280c8944a124bcb6bcede5e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 10:51:47 +0800 Subject: [PATCH 528/856] [Presto] Add bug 23324 --- src/sqlancer/presto/PrestoBugs.java | 3 +++ src/sqlancer/presto/PrestoErrors.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/presto/PrestoBugs.java b/src/sqlancer/presto/PrestoBugs.java index 5e347c6f5..f5e888df1 100644 --- a/src/sqlancer/presto/PrestoBugs.java +++ b/src/sqlancer/presto/PrestoBugs.java @@ -2,6 +2,9 @@ public final class PrestoBugs { + // https://github.com/prestodb/presto/issues/23324 + public static boolean bug23324 = true; + // https://github.com/prestodb/presto/issues/23613 public static boolean bug23613 = true; diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 4fcb749c3..0e000119e 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -41,7 +41,9 @@ public static List getExpressionErrors() { // TODO: check errors.add("io.airlift.slice.Slice cannot be cast to java.lang.Number"); - errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); + if (PrestoBugs.bug23324) { + errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); + } errors.add("Cannot cast java.lang.String to java.util.List"); errors.add("Unexpected subquery expression in logical plan"); From c71f88e324418fa78a608836c6cdc36355cbac84 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 11:07:01 +0800 Subject: [PATCH 529/856] [Presto] Remove invalid timezone (America/Hermosilo) --- src/sqlancer/presto/ast/PrestoConstant.java | 44 ++++++++++----------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/presto/ast/PrestoConstant.java b/src/sqlancer/presto/ast/PrestoConstant.java index b1d30a1ab..a469b983c 100644 --- a/src/sqlancer/presto/ast/PrestoConstant.java +++ b/src/sqlancer/presto/ast/PrestoConstant.java @@ -39,28 +39,28 @@ public abstract class PrestoConstant implements PrestoExpression { "America/Dominica", "America/Edmonton", "America/Eirunepe", "America/El_Salvador", "America/Ensenada", "America/Fort_Nelson", "America/Fort_Wayne", "America/Fortaleza", "America/Glace_Bay", "America/Godthab", "America/Goose_Bay", "America/Grand_Turk", "America/Grenada", "America/Guadeloupe", "America/Guatemala", - "America/Guayaquil", "America/Guyana", "America/Halifax", "America/Havana", "America/Hermosillo", - "America/Indiana/Indianapolis", "America/Indiana/Knox", "America/Indiana/Marengo", - "America/Indiana/Petersburg", "America/Indiana/Tell_City", "America/Indiana/Vevay", - "America/Indiana/Vincennes", "America/Indiana/Winamac", "America/Indianapolis", "America/Inuvik", - "America/Iqaluit", "America/Jamaica", "America/Jujuy", "America/Juneau", "America/Kentucky/Louisville", - "America/Kentucky/Monticello", "America/Knox_IN", "America/Kralendijk", "America/La_Paz", "America/Lima", - "America/Los_Angeles", "America/Louisville", "America/Lower_Princes", "America/Maceio", "America/Managua", - "America/Manaus", "America/Marigot", "America/Martinique", "America/Matamoros", "America/Mendoza", - "America/Menominee", "America/Merida", "America/Metlakatla", "America/Mexico_City", "America/Miquelon", - "America/Moncton", "America/Monterrey", "America/Montevideo", "America/Montreal", "America/Montserrat", - "America/Nassau", "America/New_York", "America/Nipigon", "America/Nome", "America/Noronha", - "America/North_Dakota/Beulah", "America/North_Dakota/Center", "America/North_Dakota/New_Salem", - "America/Nuuk", "America/Ojinaga", "America/Panama", "America/Pangnirtung", "America/Paramaribo", - "America/Phoenix", "America/Port-au-Prince", "America/Port_of_Spain", "America/Porto_Acre", - "America/Porto_Velho", "America/Puerto_Rico", "America/Punta_Arenas", "America/Rainy_River", - "America/Rankin_Inlet", "America/Recife", "America/Regina", "America/Resolute", "America/Rio_Branco", - "America/Rosario", "America/Santa_Isabel", "America/Santarem", "America/Santiago", "America/Santo_Domingo", - "America/Sao_Paulo", "America/Scoresbysund", "America/Shiprock", "America/Sitka", "America/St_Barthelemy", - "America/St_Johns", "America/St_Kitts", "America/St_Lucia", "America/St_Thomas", "America/St_Vincent", - "America/Swift_Current", "America/Tegucigalpa", "America/Thule", "America/Thunder_Bay", "America/Tijuana", - "America/Toronto", "America/Tortola", "America/Vancouver", "America/Virgin", "America/Whitehorse", - "America/Winnipeg", "America/Yakutat", "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis", + "America/Guayaquil", "America/Guyana", "America/Halifax", "America/Havana", "America/Indiana/Indianapolis", + "America/Indiana/Knox", "America/Indiana/Marengo", "America/Indiana/Petersburg", + "America/Indiana/Tell_City", "America/Indiana/Vevay", "America/Indiana/Vincennes", + "America/Indiana/Winamac", "America/Indianapolis", "America/Inuvik", "America/Iqaluit", "America/Jamaica", + "America/Jujuy", "America/Juneau", "America/Kentucky/Louisville", "America/Kentucky/Monticello", + "America/Knox_IN", "America/Kralendijk", "America/La_Paz", "America/Lima", "America/Los_Angeles", + "America/Louisville", "America/Lower_Princes", "America/Maceio", "America/Managua", "America/Manaus", + "America/Marigot", "America/Martinique", "America/Matamoros", "America/Mendoza", "America/Menominee", + "America/Merida", "America/Metlakatla", "America/Mexico_City", "America/Miquelon", "America/Moncton", + "America/Monterrey", "America/Montevideo", "America/Montreal", "America/Montserrat", "America/Nassau", + "America/New_York", "America/Nipigon", "America/Nome", "America/Noronha", "America/North_Dakota/Beulah", + "America/North_Dakota/Center", "America/North_Dakota/New_Salem", "America/Nuuk", "America/Ojinaga", + "America/Panama", "America/Pangnirtung", "America/Paramaribo", "America/Phoenix", "America/Port-au-Prince", + "America/Port_of_Spain", "America/Porto_Acre", "America/Porto_Velho", "America/Puerto_Rico", + "America/Punta_Arenas", "America/Rainy_River", "America/Rankin_Inlet", "America/Recife", "America/Regina", + "America/Resolute", "America/Rio_Branco", "America/Rosario", "America/Santa_Isabel", "America/Santarem", + "America/Santiago", "America/Santo_Domingo", "America/Sao_Paulo", "America/Scoresbysund", + "America/Shiprock", "America/Sitka", "America/St_Barthelemy", "America/St_Johns", "America/St_Kitts", + "America/St_Lucia", "America/St_Thomas", "America/St_Vincent", "America/Swift_Current", + "America/Tegucigalpa", "America/Thule", "America/Thunder_Bay", "America/Tijuana", "America/Toronto", + "America/Tortola", "America/Vancouver", "America/Virgin", "America/Whitehorse", "America/Winnipeg", + "America/Yakutat", "America/Yellowknife", "Antarctica/Casey", "Antarctica/Davis", "Antarctica/DumontDUrville", "Antarctica/Macquarie", "Antarctica/Mawson", "Antarctica/McMurdo", "Antarctica/Palmer", "Antarctica/Rothera", "Antarctica/South_Pole", "Antarctica/Syowa", "Antarctica/Troll", "Antarctica/Vostok", "Arctic/Longyearbyen", "Asia/Aden", "Asia/Almaty", "Asia/Amman", "Asia/Anadyr", From 423c01ac0130fae96c3ef0bee9f4475eec0a1f5d Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 13:33:08 +0800 Subject: [PATCH 530/856] [HSQLDB] Implement expression generator interfaces --- src/sqlancer/hsqldb/ast/HSQLDBExpression.java | 5 +- src/sqlancer/hsqldb/ast/HSQLDBJoin.java | 51 +++------- src/sqlancer/hsqldb/ast/HSQLDBSelect.java | 26 ++++- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 98 ++++++++++++++++++- 4 files changed, 137 insertions(+), 43 deletions(-) diff --git a/src/sqlancer/hsqldb/ast/HSQLDBExpression.java b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java index b5359c11c..d066b4359 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBExpression.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBExpression.java @@ -1,4 +1,7 @@ package sqlancer.hsqldb.ast; -public interface HSQLDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; + +public interface HSQLDBExpression extends Expression { } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java index 595384edd..a895e1b20 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBJoin.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBJoin.java @@ -1,19 +1,16 @@ package sqlancer.hsqldb.ast; -import java.util.ArrayList; -import java.util.List; - import sqlancer.Randomly; -import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; -import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; +import sqlancer.common.ast.newast.Join; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; -public class HSQLDBJoin implements HSQLDBExpression { +public class HSQLDBJoin implements HSQLDBExpression, Join { private final HSQLDBTableReference leftTable; private final HSQLDBTableReference rightTable; private final JoinType joinType; - private final HSQLDBExpression onCondition; + private HSQLDBExpression onCondition; private OuterType outerType; public enum JoinType { @@ -64,37 +61,6 @@ public OuterType getOuterType() { return outerType; } - public static List getJoins(List tableList, HSQLDBGlobalState globalState) { - List joinExpressions = new ArrayList<>(); - while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { - HSQLDBTableReference leftTable = tableList.remove(0); - HSQLDBTableReference rightTable = tableList.remove(0); - List columns = new ArrayList<>(leftTable.getTable().getColumns()); - columns.addAll(rightTable.getTable().getColumns()); - HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(globalState).setColumns(columns); - switch (HSQLDBJoin.JoinType.getRandom()) { - case INNER: - joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTable, rightTable, - joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - case NATURAL: - joinExpressions.add(HSQLDBJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); - break; - case LEFT: - joinExpressions.add(HSQLDBJoin.createLeftOuterJoin(leftTable, rightTable, - joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - case RIGHT: - joinExpressions.add(HSQLDBJoin.createRightOuterJoin(leftTable, rightTable, - joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); - break; - default: - throw new AssertionError(); - } - } - return joinExpressions; - } - public static HSQLDBJoin createRightOuterJoin(HSQLDBTableReference left, HSQLDBTableReference right, HSQLDBExpression predicate) { return new HSQLDBJoin(left, right, JoinType.RIGHT, predicate); @@ -110,11 +76,16 @@ public static HSQLDBJoin createInnerJoin(HSQLDBTableReference left, HSQLDBTableR return new HSQLDBJoin(left, right, JoinType.INNER, predicate); } - public static HSQLDBExpression createNaturalJoin(HSQLDBTableReference left, HSQLDBTableReference right, + public static HSQLDBJoin createNaturalJoin(HSQLDBTableReference left, HSQLDBTableReference right, OuterType naturalJoinType) { HSQLDBJoin join = new HSQLDBJoin(left, right, JoinType.NATURAL, null); join.setOuterType(naturalJoinType); return join; } + @Override + public void setOnClause(HSQLDBExpression onClause) { + onCondition = onClause; + } + } diff --git a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java index c75b0b88a..d58275d1e 100644 --- a/src/sqlancer/hsqldb/ast/HSQLDBSelect.java +++ b/src/sqlancer/hsqldb/ast/HSQLDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.hsqldb.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.HSQLDBToStringVisitor; -public class HSQLDBSelect extends SelectBase implements HSQLDBExpression { +public class HSQLDBSelect extends SelectBase + implements HSQLDBExpression, Select { private boolean isDistinct; @@ -14,4 +22,20 @@ public boolean isDistinct() { return isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (HSQLDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (HSQLDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return HSQLDBToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index b5304c890..4426c0c6c 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -7,18 +7,28 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; import sqlancer.hsqldb.ast.HSQLDBBinaryOperation; import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBConstant; import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.ast.HSQLDBTableReference; import sqlancer.hsqldb.ast.HSQLDBUnaryPostfixOperation; import sqlancer.hsqldb.ast.HSQLDBUnaryPrefixOperation; public final class HSQLDBExpressionGenerator extends - TypedExpressionGenerator { + TypedExpressionGenerator + implements NoRECGenerator { + + List tables; private enum Expression { BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC; @@ -193,4 +203,90 @@ public List generateOrderBys() { } return expressions; } + + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public HSQLDBExpression generateBooleanExpression() { + return generatePredicate(); + } + + @Override + public HSQLDBSelect generateSelect() { + return new HSQLDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List joinExpressions = new ArrayList<>(); + while (tables.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + HSQLDBTable leftTable = tables.remove(0); + HSQLDBTable rightTable = tables.remove(0); + List columns = new ArrayList<>(leftTable.getColumns()); + columns.addAll(rightTable.getColumns()); + HSQLDBExpressionGenerator joinGen = new HSQLDBExpressionGenerator(hsqldbGlobalState).setColumns(columns); + HSQLDBTableReference leftTableRef = new HSQLDBTableReference(leftTable); + HSQLDBTableReference rightTableRef = new HSQLDBTableReference(rightTable); + switch (HSQLDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(HSQLDBJoin.createInnerJoin(leftTableRef, rightTableRef, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case NATURAL: + joinExpressions.add( + HSQLDBJoin.createNaturalJoin(leftTableRef, rightTableRef, HSQLDBJoin.OuterType.getRandom())); + break; + case LEFT: + joinExpressions.add(HSQLDBJoin.createLeftOuterJoin(leftTableRef, rightTableRef, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + case RIGHT: + joinExpressions.add(HSQLDBJoin.createRightOuterJoin(leftTableRef, rightTableRef, + joinGen.generateExpression(HSQLDBSchema.HSQLDBCompositeDataType.getRandomWithoutNull()))); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new HSQLDBTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(HSQLDBSelect select, HSQLDBExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + HSQLDBColumn aggr = new HSQLDBColumn("COUNT(*)", null, null); + select.setFetchColumns(List.of(new HSQLDBColumnReference(aggr))); + } else { + List allColumns = columns.stream().map((c) -> new HSQLDBColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(HSQLDBSelect select, HSQLDBExpression whereCondition) { + HSQLDBColumn c = new HSQLDBColumn("COUNT(*) as count", null, null); + select.setFetchColumns(List.of(new HSQLDBColumnReference(c))); + select.setWhereClause(null); + return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; + } } From 5e8cd5d51f08cca6a0f15b965651178dbc1bfb7f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 13:34:34 +0800 Subject: [PATCH 531/856] [HSQLDB] Use common NoREC oracle --- .../hsqldb/test/HSQLDBNoRECOracle.java | 108 +++--------------- 1 file changed, 15 insertions(+), 93 deletions(-) diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java index e5cc786c3..0b1a3b8b8 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java @@ -1,121 +1,43 @@ package sqlancer.hsqldb.test; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.hsqldb.HSQLDBErrors; import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; import sqlancer.hsqldb.HSQLDBSchema; import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBCompositeDataType; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBDataType; import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; -import sqlancer.hsqldb.HSQLDBToStringVisitor; -import sqlancer.hsqldb.ast.HSQLDBColumnReference; import sqlancer.hsqldb.ast.HSQLDBExpression; import sqlancer.hsqldb.ast.HSQLDBJoin; import sqlancer.hsqldb.ast.HSQLDBSelect; -import sqlancer.hsqldb.ast.HSQLDBTableReference; import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -public class HSQLDBNoRECOracle extends NoRECBase implements TestOracle { +public class HSQLDBNoRECOracle implements TestOracle { - private final HSQLDBSchema s; + NoRECOracle oracle; public HSQLDBNoRECOracle(HSQLDBGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - HSQLDBErrors.addExpressionErrors(errors); + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - List tables = s.getDatabaseTablesRandomSubsetNotEmpty(); - List columns = tables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList()); - HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state).setColumns(columns); - - HSQLDBExpression randomWhereCondition = gen - .generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)); - - List tableList = tables.stream().map(t -> new HSQLDBTableReference(t)) - .collect(Collectors.toList()); - List joins = HSQLDBJoin.getJoins(tableList, state); - int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); // 禁用优化 - int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getSecondQuery(List tableList, HSQLDBExpression randomWhereCondition, - List joins) throws SQLException { - HSQLDBSelect select = new HSQLDBSelect(); - HSQLDBColumn c = new HSQLDBColumn("COUNT(*)", null, null); - select.setFetchColumns(List.of(new HSQLDBColumnReference(c))); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + HSQLDBToStringVisitor.asString(select) + ") as res"; - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getFirstQueryCount(SQLConnection con, List tableList, List columns, - HSQLDBExpression randomWhereCondition, List joins) throws SQLException { - HSQLDBSelect select = new HSQLDBSelect(); - List allColumns = columns.stream().map((c) -> new HSQLDBColumnReference(c)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new HSQLDBExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = HSQLDBToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From 9a4ce4b5639c264cf226e572613e0d8a0e69b05e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 13:36:58 +0800 Subject: [PATCH 532/856] [HSQLDB] Update expected errors --- src/sqlancer/hsqldb/HSQLDBErrors.java | 29 +++++++++------------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/hsqldb/HSQLDBErrors.java b/src/sqlancer/hsqldb/HSQLDBErrors.java index c02f8f2b9..33995a0e1 100644 --- a/src/sqlancer/hsqldb/HSQLDBErrors.java +++ b/src/sqlancer/hsqldb/HSQLDBErrors.java @@ -12,35 +12,26 @@ private HSQLDBErrors() { public static List getExpressionErrors() { ArrayList errors = new ArrayList<>(); - errors.add(""); - return errors; - } - - public static void addExpressionErrors(ExpectedErrors errors) { - errors.addAll(getExpressionErrors()); - } - - private static List getRegexErrors() { - ArrayList errors = new ArrayList<>(); - errors.add(""); + errors.add("invalid datetime format"); + errors.add("invalid character value for cast"); + errors.add("invalid ORDER BY expression"); + errors.add("data type of expression is not boolean"); + errors.add("numeric value out of range"); + errors.add("incompatible data types in combination"); + errors.add("string data, right truncation"); return errors; } - private static List getFunctionErrors() { - ArrayList errors = new ArrayList<>(); - - errors.add(""); - - return errors; + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); } public static List getInsertErrors() { ArrayList errors = new ArrayList<>(); - errors.addAll(getRegexErrors()); - errors.addAll(getFunctionErrors()); + errors.addAll(getExpressionErrors()); return errors; } From 252cb961350ac89cd60f9f190db7f9baf2f58638 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 15:14:20 +0800 Subject: [PATCH 533/856] [H2] Implement TLP where expression generator interfaces --- src/sqlancer/h2/H2ExpressionGenerator.java | 54 ++++++++++++++++++- src/sqlancer/h2/H2RandomQuerySynthesizer.java | 2 +- src/sqlancer/h2/ast/H2Expression.java | 5 +- src/sqlancer/h2/ast/H2Join.java | 17 ++++-- src/sqlancer/h2/ast/H2Select.java | 26 ++++++++- 5 files changed, 95 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/h2/H2ExpressionGenerator.java b/src/sqlancer/h2/H2ExpressionGenerator.java index d4a2240e6..3d816292c 100644 --- a/src/sqlancer/h2/H2ExpressionGenerator.java +++ b/src/sqlancer/h2/H2ExpressionGenerator.java @@ -1,12 +1,18 @@ package sqlancer.h2; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; import sqlancer.h2.H2Schema.H2CompositeDataType; import sqlancer.h2.H2Schema.H2DataType; +import sqlancer.h2.H2Schema.H2Table; import sqlancer.h2.ast.H2BetweenOperation; import sqlancer.h2.ast.H2BinaryOperation; import sqlancer.h2.ast.H2CaseOperation; @@ -15,12 +21,17 @@ import sqlancer.h2.ast.H2Constant; import sqlancer.h2.ast.H2Expression; import sqlancer.h2.ast.H2InOperation; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; +import sqlancer.h2.ast.H2TableReference; import sqlancer.h2.ast.H2UnaryPostfixOperation; import sqlancer.h2.ast.H2UnaryPrefixOperation; -public class H2ExpressionGenerator extends UntypedExpressionGenerator { +public class H2ExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { private final H2GlobalState globalState; + private List tables; public H2ExpressionGenerator(H2GlobalState globalState) { this.globalState = globalState; @@ -336,4 +347,45 @@ public H2Expression isNull(H2Expression expr) { return new H2UnaryPostfixOperation(expr, H2UnaryPostfixOperator.IS_NULL); } + @Override + public TLPWhereGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public H2Expression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public H2Select generateSelect() { + return new H2Select(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new H2TableReference(t)) + .collect(Collectors.toList()); + List joins = H2Join.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new H2TableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBoolean()) { + return List.of(new H2ColumnReference(new H2Column("*", null))); + } + return Randomly.nonEmptySubset(this.columns).stream().map(c -> new H2ColumnReference(c)) + .collect(Collectors.toList()); + } } diff --git a/src/sqlancer/h2/H2RandomQuerySynthesizer.java b/src/sqlancer/h2/H2RandomQuerySynthesizer.java index ba89bfd0b..7390b87e7 100644 --- a/src/sqlancer/h2/H2RandomQuerySynthesizer.java +++ b/src/sqlancer/h2/H2RandomQuerySynthesizer.java @@ -32,7 +32,7 @@ public static H2Select generateSelect(H2GlobalState globalState, int nrColumns) List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new H2TableReference(t)) .collect(Collectors.toList()); - List joins = H2Join.getJoins(tableList, globalState); + List joins = H2Join.getJoins(tableList, globalState); select.setJoinList(joins.stream().collect(Collectors.toList())); select.setFromList(tableList.stream().collect(Collectors.toList())); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/h2/ast/H2Expression.java b/src/sqlancer/h2/ast/H2Expression.java index ed41cdfab..afd114c91 100644 --- a/src/sqlancer/h2/ast/H2Expression.java +++ b/src/sqlancer/h2/ast/H2Expression.java @@ -1,5 +1,8 @@ package sqlancer.h2.ast; -public interface H2Expression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.h2.H2Schema.H2Column; + +public interface H2Expression extends Expression { } diff --git a/src/sqlancer/h2/ast/H2Join.java b/src/sqlancer/h2/ast/H2Join.java index e777e4fc9..6aafd1de9 100644 --- a/src/sqlancer/h2/ast/H2Join.java +++ b/src/sqlancer/h2/ast/H2Join.java @@ -4,16 +4,18 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.h2.H2ExpressionGenerator; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; -public class H2Join implements H2Expression { +public class H2Join implements H2Expression, Join { private final H2TableReference leftTable; private final H2TableReference rightTable; private final JoinType joinType; - private final H2Expression onCondition; + private H2Expression onCondition; public enum JoinType { INNER, CROSS, NATURAL, LEFT, RIGHT; @@ -47,8 +49,8 @@ public H2Expression getOnCondition() { return onCondition; } - public static List getJoins(List tableList, H2GlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, H2GlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { H2TableReference leftTable = tableList.remove(0); H2TableReference rightTable = tableList.remove(0); @@ -91,8 +93,13 @@ public static H2Join createInnerJoin(H2TableReference left, H2TableReference rig return new H2Join(left, right, JoinType.INNER, predicate); } - public static H2Expression createNaturalJoin(H2TableReference left, H2TableReference right) { + public static H2Join createNaturalJoin(H2TableReference left, H2TableReference right) { return new H2Join(left, right, JoinType.NATURAL, null); } + @Override + public void setOnClause(H2Expression onClause) { + onCondition = onClause; + } + } diff --git a/src/sqlancer/h2/ast/H2Select.java b/src/sqlancer/h2/ast/H2Select.java index 9658ebca0..e2f6f8519 100644 --- a/src/sqlancer/h2/ast/H2Select.java +++ b/src/sqlancer/h2/ast/H2Select.java @@ -1,7 +1,31 @@ package sqlancer.h2.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; +import sqlancer.h2.H2ToStringVisitor; + +public class H2Select extends SelectBase + implements H2Expression, Select { + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (H2Expression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } -public class H2Select extends SelectBase implements H2Expression { + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (H2Join) e).collect(Collectors.toList()); + } + @Override + public String asString() { + return H2ToStringVisitor.asString(this); + } } From 05fb96ebdbd1875d851c0e181274993ebdb9d0b8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 15:14:48 +0800 Subject: [PATCH 534/856] [H2] Use common TLP where oracle --- src/sqlancer/h2/H2QueryPartitioningBase.java | 62 ------------------- .../h2/H2QueryPartitioningWhereTester.java | 47 ++++++-------- 2 files changed, 20 insertions(+), 89 deletions(-) delete mode 100644 src/sqlancer/h2/H2QueryPartitioningBase.java diff --git a/src/sqlancer/h2/H2QueryPartitioningBase.java b/src/sqlancer/h2/H2QueryPartitioningBase.java deleted file mode 100644 index 34351f58f..000000000 --- a/src/sqlancer/h2/H2QueryPartitioningBase.java +++ /dev/null @@ -1,62 +0,0 @@ -package sqlancer.h2; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.h2.H2Provider.H2GlobalState; -import sqlancer.h2.H2Schema.H2Column; -import sqlancer.h2.H2Schema.H2Table; -import sqlancer.h2.H2Schema.H2Tables; -import sqlancer.h2.ast.H2ColumnReference; -import sqlancer.h2.ast.H2Expression; -import sqlancer.h2.ast.H2Join; -import sqlancer.h2.ast.H2Select; -import sqlancer.h2.ast.H2TableReference; - -public class H2QueryPartitioningBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - H2Schema s; - H2Tables targetTables; - H2ExpressionGenerator gen; - H2Select select; - - public H2QueryPartitioningBase(H2GlobalState state) { - super(state); - H2Errors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new H2ExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new H2Select(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List tableList = tables.stream().map(t -> new H2TableReference(t)) - .collect(Collectors.toList()); - List joins = H2Join.getJoins(tableList, state); - select.setJoinList(joins.stream().collect(Collectors.toList())); - select.setFromList(tableList.stream().collect(Collectors.toList())); - select.setWhereClause(null); - } - - List generateFetchColumns() { - List columns = new ArrayList<>(); - columns.add(new H2ColumnReference(new H2Column("*", null))); - return columns; - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java index cc09dea7a..f1bb73dfa 100644 --- a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java +++ b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java @@ -1,42 +1,35 @@ package sqlancer.h2; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.h2.H2Provider.H2GlobalState; +import sqlancer.h2.H2Schema.H2Column; +import sqlancer.h2.H2Schema.H2Table; +import sqlancer.h2.ast.H2Expression; +import sqlancer.h2.ast.H2Join; +import sqlancer.h2.ast.H2Select; -public class H2QueryPartitioningWhereTester extends H2QueryPartitioningBase { +public class H2QueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public H2QueryPartitioningWhereTester(H2GlobalState state) { - super(state); + H2ExpressionGenerator gen = new H2ExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(H2Errors.getExpressionErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = H2ToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = H2ToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = H2ToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = H2ToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } } From 07b28ca7e22017443b8a1268805b52d492dfbfed Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 15:15:29 +0800 Subject: [PATCH 535/856] [H2] Remove environment check in test --- .github/workflows/main.yml | 2 +- test/sqlancer/dbms/TestH2.java | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6150b8431..9434cb8e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -268,7 +268,7 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: H2_AVAILABLE=true mvn -Dtest=TestH2 test + run: mvn -Dtest=TestH2 test hsqldb: name: DBMS Tests (HSQLB) diff --git a/test/sqlancer/dbms/TestH2.java b/test/sqlancer/dbms/TestH2.java index 701f6d50b..ec6962101 100644 --- a/test/sqlancer/dbms/TestH2.java +++ b/test/sqlancer/dbms/TestH2.java @@ -1,7 +1,6 @@ package sqlancer.dbms; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; import org.junit.jupiter.api.Test; @@ -11,9 +10,6 @@ public class TestH2 { @Test public void testH2DB() { - String h2Available = System.getenv("H2_AVAILABLE"); - boolean h2DBIsAvailable = h2Available != null && h2Available.equalsIgnoreCase("true"); - assumeTrue(h2DBIsAvailable); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "h2" })); From 52cee37ce27fc754087869daad2df3df190aa1aa Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 16:53:34 +0800 Subject: [PATCH 536/856] [TiDB] Implement TLP where expression gen interface --- .../tidb/TiDBExpressionGenerator.java | 54 ++++++++++++++++++- src/sqlancer/tidb/ast/TiDBExpression.java | 5 +- src/sqlancer/tidb/ast/TiDBJoin.java | 12 +++-- src/sqlancer/tidb/ast/TiDBSelect.java | 26 ++++++++- src/sqlancer/tidb/oracle/TiDBDQPOracle.java | 2 +- 5 files changed, 92 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index a1db6fe2a..0db285284 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -3,13 +3,18 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; import sqlancer.tidb.TiDBSchema.TiDBDataType; +import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.ast.TiDBAggregate; import sqlancer.tidb.ast.TiDBAggregate.TiDBAggregateFunction; import sqlancer.tidb.ast.TiDBBinaryBitOperation; @@ -25,17 +30,22 @@ import sqlancer.tidb.ast.TiDBExpression; import sqlancer.tidb.ast.TiDBFunctionCall; import sqlancer.tidb.ast.TiDBFunctionCall.TiDBFunction; +import sqlancer.tidb.ast.TiDBJoin; import sqlancer.tidb.ast.TiDBOrderingTerm; import sqlancer.tidb.ast.TiDBRegexOperation; import sqlancer.tidb.ast.TiDBRegexOperation.TiDBRegexOperator; +import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableReference; import sqlancer.tidb.ast.TiDBUnaryPostfixOperation; import sqlancer.tidb.ast.TiDBUnaryPostfixOperation.TiDBUnaryPostfixOperator; import sqlancer.tidb.ast.TiDBUnaryPrefixOperation; import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; -public class TiDBExpressionGenerator extends UntypedExpressionGenerator { +public class TiDBExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { private final TiDBGlobalState globalState; + private List tables; public TiDBExpressionGenerator(TiDBGlobalState globalState) { this.globalState = globalState; @@ -188,4 +198,46 @@ public TiDBExpression generateConstant(TiDBDataType type) { } } + @Override + public TLPWhereGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public TiDBExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public TiDBSelect generateSelect() { + return new TiDBSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new TiDBTableReference(t)) + .collect(Collectors.toList()); + List joins = TiDBJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> ((TiDBTableReference) t).getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new TiDBTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBoolean()) { + return List.of(new TiDBColumnReference( + new TiDBColumn("*", new TiDBCompositeDataType(TiDBDataType.INT), false, false, false))); + } + return Randomly.nonEmptySubset(this.columns).stream().map(c -> new TiDBColumnReference(c)) + .collect(Collectors.toList()); + } } diff --git a/src/sqlancer/tidb/ast/TiDBExpression.java b/src/sqlancer/tidb/ast/TiDBExpression.java index 49ac9fb70..1f4921836 100644 --- a/src/sqlancer/tidb/ast/TiDBExpression.java +++ b/src/sqlancer/tidb/ast/TiDBExpression.java @@ -1,5 +1,8 @@ package sqlancer.tidb.ast; -public interface TiDBExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.tidb.TiDBSchema.TiDBColumn; + +public interface TiDBExpression extends Expression { } diff --git a/src/sqlancer/tidb/ast/TiDBJoin.java b/src/sqlancer/tidb/ast/TiDBJoin.java index f4496a736..1e9a30b63 100644 --- a/src/sqlancer/tidb/ast/TiDBJoin.java +++ b/src/sqlancer/tidb/ast/TiDBJoin.java @@ -5,11 +5,13 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBTable; -public class TiDBJoin implements TiDBExpression { +public class TiDBJoin implements TiDBExpression, Join { private final TiDBExpression leftTable; private final TiDBExpression rightTable; @@ -101,8 +103,8 @@ public NaturalJoinType getNaturalJoinType() { return outerType; } - public static List getJoins(List tableList, TiDBGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, TiDBGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBoolean()) { TiDBTableReference leftTable = (TiDBTableReference) tableList.remove(0); TiDBTableReference rightTable = (TiDBTableReference) tableList.remove(0); @@ -172,4 +174,8 @@ public void setOnCondition(TiDBExpression generateExpression) { this.onCondition = generateExpression; } + @Override + public void setOnClause(TiDBExpression onClause) { + onCondition = onClause; + } } diff --git a/src/sqlancer/tidb/ast/TiDBSelect.java b/src/sqlancer/tidb/ast/TiDBSelect.java index f812ee8e0..7da05da4c 100644 --- a/src/sqlancer/tidb/ast/TiDBSelect.java +++ b/src/sqlancer/tidb/ast/TiDBSelect.java @@ -1,8 +1,16 @@ package sqlancer.tidb.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBTable; +import sqlancer.tidb.visitor.TiDBVisitor; -public class TiDBSelect extends SelectBase implements TiDBExpression { +public class TiDBSelect extends SelectBase + implements TiDBExpression, Select { private TiDBExpression hint; @@ -14,4 +22,20 @@ public TiDBExpression getHint() { return hint; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (TiDBExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (TiDBJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return TiDBVisitor.asString(this); + } } diff --git a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java index 17b288b7d..57ee4409f 100644 --- a/src/sqlancer/tidb/oracle/TiDBDQPOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBDQPOracle.java @@ -47,7 +47,7 @@ public void check() throws SQLException { List tableList = tables.getTables().stream().map(t -> new TiDBTableReference(t)) .collect(Collectors.toList()); - List joins = TiDBJoin.getJoins(tableList, state); + List joins = TiDBJoin.getJoins(tableList, state).stream().collect(Collectors.toList()); select.setJoinList(joins); select.setFromList(tableList); if (Randomly.getBoolean()) { From ec567350c880e36660926631b3081e51dec6ba48 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 16:54:19 +0800 Subject: [PATCH 537/856] Implement reproducer for TLP-Where oracle --- .../common/oracle/TLPWhereOracle.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 5f7b3feb6..14834a62f 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -6,6 +6,7 @@ import sqlancer.ComparatorHelper; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.SQLGlobalState; import sqlancer.common.ast.newast.Expression; import sqlancer.common.ast.newast.Join; @@ -25,8 +26,43 @@ public class TLPWhereOracle, J extends Join gen; private final ExpectedErrors errors; + private Reproducer reproducer; private String generatedQueryString; + private class TLPWhereReproducer implements Reproducer { + final String firstQueryString; + final String secondQueryString; + final String thirdQueryString; + final String originalQueryString; + final List resultSet; + final boolean orderBy; + + TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, + String originalQueryString, List resultSet, boolean orderBy) { + this.firstQueryString = firstQueryString; + this.secondQueryString = secondQueryString; + this.thirdQueryString = thirdQueryString; + this.originalQueryString = originalQueryString; + this.resultSet = resultSet; + this.orderBy = orderBy; + } + + @Override + public boolean bugStillTriggers(G globalState) { + try { + List combinedString1 = new ArrayList<>(); + List secondResultSet1 = ComparatorHelper.getCombinedResultSet(firstQueryString, + secondQueryString, thirdQueryString, combinedString1, !orderBy, globalState, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet1, originalQueryString, + combinedString1, globalState); + } catch (AssertionError triggeredError) { + return true; + } catch (SQLException ignored) { + } + return false; + } + } + public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -38,6 +74,7 @@ public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErr @Override public void check() throws SQLException { + reproducer = null; S s = state.getSchema(); AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(s); gen = gen.setTablesAndColumns(targetTables); @@ -75,6 +112,14 @@ public void check() throws SQLException { ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); + + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, + firstResultSet, orderBy); + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; } @Override From 7e558b54378ff0c01e6466f5512541440f60a680 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 13 Sep 2024 17:40:31 +0800 Subject: [PATCH 538/856] Add expected errors for NoREC oracle --- src/sqlancer/duckdb/DuckDBErrors.java | 17 +++++++++++++---- src/sqlancer/duckdb/test/DuckDBNoRECOracle.java | 3 ++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 1cc7bd96b..6e02a58f6 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.regex.Pattern; import sqlancer.common.query.ExpectedErrors; @@ -58,12 +59,20 @@ public static List getExpressionErrors() { return errors; } + public static List getExpressionErrorsRegex() { + ArrayList errors = new ArrayList<>(); + + errors.add(Pattern.compile("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause")); + errors.add(Pattern.compile("Binder Error: Cannot mix values of type .* and .* in CASE expression")); + errors.add(Pattern.compile("Cannot mix values of type .* and .* in COALESCE operator")); + errors.add(Pattern.compile("Cannot compare values of type .* and type .*")); + + return errors; + } + public static void addExpressionErrors(ExpectedErrors errors) { errors.addAll(getExpressionErrors()); - errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in BETWEEN clause"); - errors.addRegexString("Binder Error: Cannot mix values of type .* and .* in CASE expression"); - errors.addRegexString("Cannot mix values of type .* and .* in COALESCE operator"); - errors.addRegexString("Cannot compare values of type .* and type .*"); + errors.addAllRegexes(getExpressionErrorsRegex()); } private static List getRegexErrors() { diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java index 4c57c0920..4096714ac 100644 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java @@ -23,7 +23,8 @@ public class DuckDBNoRECOracle implements TestOracle { public DuckDBNoRECOracle(DuckDBGlobalState globalState) { DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); ExpectedErrors errors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) - .with("canceling statement due to statement timeout").build(); + .withRegex(DuckDBErrors.getExpressionErrorsRegex()).with("canceling statement due to statement timeout") + .build(); this.oracle = new NoRECOracle<>(globalState, gen, errors); } From cf9492ddcb803790f9572dc783b1991987bfdb4e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 16:54:59 +0800 Subject: [PATCH 539/856] [TiDB] Use common NoREC oracle --- src/sqlancer/tidb/oracle/TiDBTLPBase.java | 2 +- .../tidb/oracle/TiDBTLPWhereOracle.java | 88 ++++--------------- test/sqlancer/dbms/TestConfig.java | 1 + 3 files changed, 20 insertions(+), 71 deletions(-) diff --git a/src/sqlancer/tidb/oracle/TiDBTLPBase.java b/src/sqlancer/tidb/oracle/TiDBTLPBase.java index 9e86ffd12..d23e2c4c0 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPBase.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPBase.java @@ -50,7 +50,7 @@ public void check() throws SQLException { List tableList = tables.stream().map(t -> new TiDBTableReference(t)) .collect(Collectors.toList()); - List joins = TiDBJoin.getJoins(tableList, state); + List joins = TiDBJoin.getJoins(tableList, state).stream().collect(Collectors.toList()); select.setJoinList(joins); select.setFromList(tableList); select.setWhereClause(null); diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java index e437b2856..61ec16788 100644 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java @@ -1,96 +1,44 @@ package sqlancer.tidb.oracle; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.tidb.TiDBErrors; +import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.visitor.TiDBVisitor; +import sqlancer.tidb.TiDBSchema; +import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBTable; +import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBSelect; -public class TiDBTLPWhereOracle extends TiDBTLPBase { +public class TiDBTLPWhereOracle implements TestOracle { - private String generatedQueryString; - private Reproducer reproducer; + private final TLPWhereOracle oracle; public TiDBTLPWhereOracle(TiDBGlobalState state) { - super(state); - TiDBErrors.addExpressionErrors(errors); - } - - private class TiDBTLPWhereReproducer implements Reproducer { - final String firstQueryString; - final String secondQueryString; - final String thirdQueryString; - final String originalQueryString; - final List resultSet; - final boolean orderBy; - - TiDBTLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, - String originalQueryString, List resultSet, boolean orderBy) { - this.firstQueryString = firstQueryString; - this.secondQueryString = secondQueryString; - this.thirdQueryString = thirdQueryString; - this.originalQueryString = originalQueryString; - this.resultSet = resultSet; - this.orderBy = orderBy; - } + TiDBExpressionGenerator gen = new TiDBExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); - @Override - public boolean bugStillTriggers(TiDBGlobalState globalState) { - try { - List combinedString1 = new ArrayList<>(); - List secondResultSet1 = ComparatorHelper.getCombinedResultSet(firstQueryString, - secondQueryString, thirdQueryString, combinedString1, !orderBy, globalState, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet1, originalQueryString, - combinedString1, globalState); - } catch (AssertionError triggeredError) { - return true; - } catch (SQLException ignored) { - } - return false; - } + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - reproducer = null; - super.check(); - select.setWhereClause(null); - String originalQueryString = TiDBVisitor.asString(select); - generatedQueryString = originalQueryString; - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = TiDBVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = TiDBVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = TiDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - reproducer = new TiDBTLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, - originalQueryString, resultSet, orderBy); + oracle.check(); } @Override public String getLastQueryString() { - return generatedQueryString; + return oracle.getLastQueryString(); } @Override public Reproducer getLastReproducer() { - return reproducer; + return oracle.getLastReproducer(); } - } diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 6fbf12dc7..8759ff5d5 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -15,6 +15,7 @@ public class TestConfig { public static final String OCEANBASE_ENV = "OCEANBASE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static final String PRESTO_ENV = "PRESTO_AVAILABLE"; + public static final String TIDB_ENV = "TIDB_AVAILABLE"; public static final String YUGABYTE_ENV = "YUGABYTE_AVAILABLE"; public static boolean isEnvironmentTrue(String key) { From 21689fc78c33781c21d5088748785fadeb3f9e84 Mon Sep 17 00:00:00 2001 From: John Keith Hohm Date: Fri, 13 Sep 2024 12:13:33 -0400 Subject: [PATCH 540/856] Bump SQLite3 to version 3.46.1.0, recognize new bad JSON path error * bump SQLite to 3.46.1.0 (from 3.44.0.0) * recognize `bad JSON path`, the new `JSON path error` since https://sqlite.org/src/info/7f0c79b94e8f55e5 --- pom.xml | 2 +- src/sqlancer/sqlite3/SQLite3Errors.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9bee83485..16ad8c59f 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ org.xerial sqlite-jdbc - 3.40.0.0 + 3.46.1.0 mysql diff --git a/src/sqlancer/sqlite3/SQLite3Errors.java b/src/sqlancer/sqlite3/SQLite3Errors.java index 406729bfb..96e45c549 100644 --- a/src/sqlancer/sqlite3/SQLite3Errors.java +++ b/src/sqlancer/sqlite3/SQLite3Errors.java @@ -56,6 +56,7 @@ public static List getExpectedExpressionErrors() { errors.add("malformed JSON"); errors.add("JSON cannot hold BLOB values"); errors.add("JSON path error"); + errors.add("bad JSON path"); errors.add("json_insert() needs an odd number of arguments"); errors.add("json_object() labels must be TEXT"); errors.add("json_object() requires an even number of arguments"); From 5bcd4ce0e4de0b34e9c1f9765843cf8d8a90377c Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 10:16:26 +0800 Subject: [PATCH 541/856] Update DuckDB version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 129908302..49d8a2ddf 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 1.0.0 + 1.1.0 com.facebook.presto From a330e8e82a6d3de39562c3167e5949a4e2c36705 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 11:22:17 +0800 Subject: [PATCH 542/856] Update expected errors --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 6e02a58f6..5141ca0b1 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -51,6 +51,7 @@ public static List getExpressionErrors() { // removed column errors.add("Contents of view were altered: types don't match!"); errors.add("Not implemented: ROUND(DECIMAL, INTEGER) with non-constant precision is not supported"); + errors.add("ORDER BY non-integer literal has no effect"); // timestamp errors.add("Cannot subtract infinite timestamps"); From 41df4fbba0d413ef5e0ec1ffd4da401189087981 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 11:22:38 +0800 Subject: [PATCH 543/856] Add bug 13933 --- src/sqlancer/duckdb/DuckDBBugs.java | 11 +++++++++++ src/sqlancer/duckdb/DuckDBErrors.java | 6 ++++++ 2 files changed, 17 insertions(+) create mode 100644 src/sqlancer/duckdb/DuckDBBugs.java diff --git a/src/sqlancer/duckdb/DuckDBBugs.java b/src/sqlancer/duckdb/DuckDBBugs.java new file mode 100644 index 000000000..001deec80 --- /dev/null +++ b/src/sqlancer/duckdb/DuckDBBugs.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb; + +public final class DuckDBBugs { + + // https://github.com/duckdb/duckdb/issues/13933 + public static boolean bug13933 = true; + + private DuckDBBugs() { + } + +} diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 5141ca0b1..bea136670 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -57,6 +57,12 @@ public static List getExpressionErrors() { errors.add("Cannot subtract infinite timestamps"); errors.add("Timestamp difference is out of bounds"); + if (DuckDBBugs.bug13933) { + errors.add("INTERNAL Error: Unimplemented type for normalify"); + errors.add( + "FATAL Error: Failed: database has been invalidated because of a previous fatal error. The database must be restarted prior to being used again"); + } + return errors; } From 97b47768e73d56bb98fe89f8234b093fcec2f8e0 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 15:15:40 +0800 Subject: [PATCH 544/856] [Presto] Implement NoREC expression generator interfaces --- src/sqlancer/presto/ast/PrestoExpression.java | 5 +- src/sqlancer/presto/ast/PrestoJoin.java | 15 +++- src/sqlancer/presto/ast/PrestoSelect.java | 26 +++++- .../gen/PrestoRandomQuerySynthesizer.java | 3 +- .../gen/PrestoTypedExpressionGenerator.java | 87 ++++++++++++++++++- .../test/PrestoQueryPartitioningBase.java | 2 +- 6 files changed, 128 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/presto/ast/PrestoExpression.java b/src/sqlancer/presto/ast/PrestoExpression.java index 143eb6823..52e0ed784 100644 --- a/src/sqlancer/presto/ast/PrestoExpression.java +++ b/src/sqlancer/presto/ast/PrestoExpression.java @@ -1,5 +1,8 @@ package sqlancer.presto.ast; -public interface PrestoExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.presto.PrestoSchema.PrestoColumn; + +public interface PrestoExpression extends Expression { } diff --git a/src/sqlancer/presto/ast/PrestoJoin.java b/src/sqlancer/presto/ast/PrestoJoin.java index 0732e55be..3fe82fc37 100644 --- a/src/sqlancer/presto/ast/PrestoJoin.java +++ b/src/sqlancer/presto/ast/PrestoJoin.java @@ -4,17 +4,19 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoJoin implements PrestoExpression { +public class PrestoJoin implements PrestoExpression, Join { private final PrestoTableReference leftTable; private final PrestoTableReference rightTable; private final JoinType joinType; - private final PrestoExpression onCondition; + private PrestoExpression onCondition; private OuterType outerType; public PrestoJoin(PrestoTableReference leftTable, PrestoTableReference rightTable, JoinType joinType, @@ -25,8 +27,8 @@ public PrestoJoin(PrestoTableReference leftTable, PrestoTableReference rightTabl this.onCondition = whereCondition; } - public static List getJoins(List tableList, PrestoGlobalState globalState) { - List joinExpressions = new ArrayList<>(); + public static List getJoins(List tableList, PrestoGlobalState globalState) { + List joinExpressions = new ArrayList<>(); while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { PrestoTableReference leftTable = tableList.remove(0); PrestoTableReference rightTable = tableList.remove(0); @@ -110,4 +112,9 @@ public static OuterType getRandom() { } } + @Override + public void setOnClause(PrestoExpression onClause) { + onCondition = onClause; + } + } diff --git a/src/sqlancer/presto/ast/PrestoSelect.java b/src/sqlancer/presto/ast/PrestoSelect.java index 328c154c4..f1eb50186 100644 --- a/src/sqlancer/presto/ast/PrestoSelect.java +++ b/src/sqlancer/presto/ast/PrestoSelect.java @@ -1,8 +1,16 @@ package sqlancer.presto.ast; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; -public class PrestoSelect extends SelectBase implements PrestoExpression { +public class PrestoSelect extends SelectBase + implements PrestoExpression, Select { private boolean isDistinct; @@ -14,4 +22,20 @@ public void setDistinct(boolean isDistinct) { this.isDistinct = isDistinct; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (PrestoExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (PrestoJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return PrestoToStringVisitor.asString(this); + } } diff --git a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java index 27a54d926..5c3c0db82 100644 --- a/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java +++ b/src/sqlancer/presto/gen/PrestoRandomQuerySynthesizer.java @@ -42,7 +42,8 @@ public static PrestoSelect generateSelect(PrestoGlobalState globalState, int nrC List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new PrestoTableReference(t)) .collect(Collectors.toList()); - List joins = PrestoJoin.getJoins(tableList, globalState); + List joins = PrestoJoin.getJoins(tableList, globalState).stream() + .collect(Collectors.toList()); select.setJoinList(new ArrayList<>(joins)); select.setFromList(new ArrayList<>(tableList)); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java index 712ebdcc3..42e3b8691 100644 --- a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -9,10 +9,17 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; +import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; +import sqlancer.presto.PrestoSchema.PrestoDataType; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.PrestoToStringVisitor; import sqlancer.presto.ast.PrestoAggregateFunction; import sqlancer.presto.ast.PrestoAtTimeZoneOperator; import sqlancer.presto.ast.PrestoBetweenOperation; @@ -29,6 +36,7 @@ import sqlancer.presto.ast.PrestoMultiValuedComparison; import sqlancer.presto.ast.PrestoMultiValuedComparisonOperator; import sqlancer.presto.ast.PrestoMultiValuedComparisonType; +import sqlancer.presto.ast.PrestoPostfixText; import sqlancer.presto.ast.PrestoQuantifiedComparison; import sqlancer.presto.ast.PrestoSelect; import sqlancer.presto.ast.PrestoTableReference; @@ -37,11 +45,13 @@ import sqlancer.presto.ast.PrestoUnaryPrefixOperation; public final class PrestoTypedExpressionGenerator extends - TypedExpressionGenerator { + TypedExpressionGenerator + implements NoRECGenerator { private final Randomly randomly; private final PrestoGlobalState globalState; private final int maxDepth; + private List tables; public PrestoTypedExpressionGenerator(PrestoGlobalState globalState) { this.globalState = globalState; @@ -338,7 +348,8 @@ private PrestoSelect generateSubquery(List columns) { if (Randomly.getBooleanWithSmallProbability()) { select.setOrderByClauses(typedExpressionGenerator.generateOrderBys()); } - List joins = PrestoJoin.getJoins(tableList, globalState); + List joins = PrestoJoin.getJoins(tableList, globalState).stream() + .collect(Collectors.toList()); select.setJoinList(joins); return select; } @@ -791,4 +802,76 @@ private enum Expression { BINARY_LOGICAL, BINARY_COMPARISON, BINARY_ARITHMETIC } + @Override + public NoRECGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public PrestoExpression generateBooleanExpression() { + return generateExpression( + PrestoSchema.PrestoCompositeDataType.fromDataType(PrestoSchema.PrestoDataType.BOOLEAN), + randomly.getInteger(0, maxDepth)); + } + + @Override + public PrestoSelect generateSelect() { + return new PrestoSelect(); + } + + @Override + public List getRandomJoinClauses() { + List tableList = tables.stream().map(t -> new PrestoTableReference(t)) + .collect(Collectors.toList()); + List joins = PrestoJoin.getJoins(tableList, globalState); + tables = tableList.stream().map(t -> t.getTable()).collect(Collectors.toList()); + return joins; + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new PrestoTableReference(t)).collect(Collectors.toList()); + } + + @Override + public String generateOptimizedQueryString(PrestoSelect select, PrestoExpression whereCondition, + boolean shouldUseAggregate) { + if (shouldUseAggregate) { + PrestoFunctionNode aggr = new PrestoFunctionNode<>( + List.of(new PrestoColumnReference(new PrestoColumn("*", + new PrestoCompositeDataType(PrestoDataType.INT, 0, 0), false, false))), + PrestoAggregateFunction.COUNT); + select.setFetchColumns(List.of(aggr)); + + } else { + List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); + select.setFetchColumns(allColumns); + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(generateOrderBys()); + } + } + select.setWhereClause(whereCondition); + + return select.asString(); + } + + @Override + public String generateUnoptimizedQueryString(PrestoSelect select, PrestoExpression whereCondition) { + PrestoExpression asText = new PrestoPostfixText( + + new PrestoCastFunction( + new PrestoPostfixText(whereCondition, + " IS NOT NULL AND " + PrestoToStringVisitor.asString(whereCondition)), + new PrestoCompositeDataType(PrestoDataType.INT, 8, 0)), + "as count"); + + select.setFetchColumns(List.of(asText)); + select.setWhereClause(null); + return "SELECT SUM(count) FROM (" + PrestoToStringVisitor.asString(select) + ") as res"; + } } diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java index 9ac6ef9f4..0e80d5c41 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningBase.java @@ -63,7 +63,7 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new PrestoTableReference(t)) .collect(Collectors.toList()); - List joins = PrestoJoin.getJoins(tableList, state); + List joins = PrestoJoin.getJoins(tableList, state).stream().collect(Collectors.toList()); select.setJoinList(new ArrayList<>(joins)); select.setFromList(new ArrayList<>(tableList)); select.setWhereClause(null); From 8dbe1f02da1167c7b6c791613bc832d0c58bf3b2 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 15:16:12 +0800 Subject: [PATCH 545/856] [Presto] Use common NoREC oracle --- .../presto/test/PrestoNoRECOracle.java | 131 +++--------------- 1 file changed, 16 insertions(+), 115 deletions(-) diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java index e9919e4d0..6ef5e68ce 100644 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ b/src/sqlancer/presto/test/PrestoNoRECOracle.java @@ -1,143 +1,44 @@ package sqlancer.presto.test; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLConnection; -import sqlancer.common.oracle.NoRECBase; +import sqlancer.Reproducer; +import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.query.ExpectedErrors; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; import sqlancer.presto.PrestoSchema.PrestoColumn; -import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; -import sqlancer.presto.PrestoSchema.PrestoDataType; import sqlancer.presto.PrestoSchema.PrestoTable; -import sqlancer.presto.PrestoSchema.PrestoTables; -import sqlancer.presto.PrestoToStringVisitor; -import sqlancer.presto.ast.PrestoCastFunction; -import sqlancer.presto.ast.PrestoColumnReference; import sqlancer.presto.ast.PrestoExpression; import sqlancer.presto.ast.PrestoJoin; -import sqlancer.presto.ast.PrestoPostfixText; import sqlancer.presto.ast.PrestoSelect; -import sqlancer.presto.ast.PrestoTableReference; import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoNoRECOracle extends NoRECBase implements TestOracle { +public class PrestoNoRECOracle implements TestOracle { - private final PrestoSchema s; + NoRECOracle oracle; public PrestoNoRECOracle(PrestoGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - PrestoErrors.addExpressionErrors(errors); + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + this.oracle = new NoRECOracle<>(globalState, gen, errors); } @Override public void check() throws SQLException { - PrestoTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - - List tables = randomTables.getTables(); - - List tableList = tables.stream().map(t -> new PrestoTableReference(t)) - .collect(Collectors.toList()); - List joins = PrestoJoin.getJoins(tableList, state); - - PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(state).setColumns(columns); - PrestoExpression randomWhereCondition = gen.generatePredicate(); - int secondCount = getSecondQuery(new ArrayList<>(tableList), randomWhereCondition, joins); - - int firstCount = getFirstQueryCount(con, new ArrayList<>(tableList), columns, randomWhereCondition, joins); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - throw new AssertionError( - optimizedQueryString + "; -- " + firstCount + "\n" + unoptimizedQueryString + " -- " + secondCount); - } + oracle.check(); } - private int getSecondQuery(List tableList, PrestoExpression randomWhereCondition, - List joins) throws SQLException { - PrestoSelect select = new PrestoSelect(); - - PrestoExpression asText = new PrestoPostfixText( - - new PrestoCastFunction( - new PrestoPostfixText(randomWhereCondition, - " IS NOT NULL AND " + PrestoToStringVisitor.asString(randomWhereCondition)), - new PrestoCompositeDataType(PrestoDataType.INT, 8, 0)), - "as count"); - - select.setFetchColumns(List.of(asText)); - select.setFromList(tableList); - select.setJoinList(joins); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + PrestoToStringVisitor.asString(select) + ") as res"; - - errors.add("canceling statement due to statement timeout"); - SQLQueryAdapter q = new SQLQueryAdapter(unoptimizedQueryString, errors, false, false); - SQLancerResultSet rs; - try { - rs = q.executeAndGetLogged(state); - } catch (Exception e) { - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - try { - if (rs.next()) { - secondCount += rs.getLong(1); - } - } catch (SQLException e) { - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } else { - throw e; - } - } - rs.close(); - return secondCount; + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } - private int getFirstQueryCount(SQLConnection con, List tableList, List columns, - PrestoExpression randomWhereCondition, List joins) { - PrestoSelect select = new PrestoSelect(); - List allColumns = columns.stream().map((c) -> new PrestoColumnReference(c)) - .collect(Collectors.toList()); - select.setFetchColumns(allColumns); - select.setFromList(tableList); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new PrestoTypedExpressionGenerator(state).setColumns(columns).generateOrderBys()); - } - select.setJoinList(joins); - int firstCount = 0; - try (Statement stat = con.createStatement()) { - optimizedQueryString = PrestoToStringVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - try (ResultSet rs = stat.executeQuery(optimizedQueryString)) { - while (rs.next()) { - firstCount++; - } - } - } catch (SQLException e) { - throw new IgnoreMeException(); - } - return firstCount; + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - } From b5c5d710ce1698ccaa734fe72302ecc1f87dd7dc Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 14 Sep 2024 15:16:22 +0800 Subject: [PATCH 546/856] Update SQL adapter options for NoREC oracle --- src/sqlancer/common/oracle/NoRECOracle.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 530488296..caf3dff87 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -121,7 +121,7 @@ public Reproducer getLastReproducer() { } private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState state) { - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { @@ -146,7 +146,7 @@ private int countRows(String queryString, ExpectedErrors errors, SQLGlobalState< } private int extractCounts(String queryString, ExpectedErrors errors, SQLGlobalState state) { - SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors); + SQLQueryAdapter q = new SQLQueryAdapter(queryString, errors, false, false); int count = 0; try (SQLancerResultSet rs = q.executeAndGet(state)) { if (rs == null) { From fab83b92b14dba5896015a1266c1565beff064b7 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 17:13:03 +0800 Subject: [PATCH 547/856] [HSQLDB] Implement TLP-where generator interface --- .../hsqldb/gen/HSQLDBExpressionGenerator.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java index 4426c0c6c..be73d1cb1 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBExpressionGenerator.java @@ -8,6 +8,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.hsqldb.HSQLDBProvider; @@ -26,7 +27,8 @@ public final class HSQLDBExpressionGenerator extends TypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { List tables; @@ -205,8 +207,7 @@ public List generateOrderBys() { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public HSQLDBExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -289,4 +290,13 @@ public String generateUnoptimizedQueryString(HSQLDBSelect select, HSQLDBExpressi select.setWhereClause(null); return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new HSQLDBColumnReference(new HSQLDBSchema.HSQLDBColumn("*", null, null))); + } + return Randomly + .nonEmptySubset(columns.stream().map(c -> new HSQLDBColumnReference(c)).collect(Collectors.toList())); + } } From f41e801bb76217ce2f12253fae1e875d33c4d79f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 17:13:55 +0800 Subject: [PATCH 548/856] [HSQLDB] Use common TLP-where oracle --- .../test/HSQLDBQueryPartitioningBase.java | 72 ------------------- .../HSQLDBQueryPartitioningWhereTester.java | 53 +++++++------- 2 files changed, 25 insertions(+), 100 deletions(-) delete mode 100644 src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java deleted file mode 100644 index b428fd02f..000000000 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningBase.java +++ /dev/null @@ -1,72 +0,0 @@ -package sqlancer.hsqldb.test; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.hsqldb.HSQLDBProvider; -import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.HSQLDBToStringVisitor; -import sqlancer.hsqldb.ast.HSQLDBColumnReference; -import sqlancer.hsqldb.ast.HSQLDBExpression; -//import sqlancer.hsqldb.ast.HSQLDBJoin; -import sqlancer.hsqldb.ast.HSQLDBSelect; -import sqlancer.hsqldb.ast.HSQLDBTableReference; -import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; - -public class HSQLDBQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - HSQLDBSelect select; - HSQLDBExpressionGenerator expressionGenerator; - - HSQLDBSchema schema; - - List targetTables; - - public HSQLDBQueryPartitioningBase(HSQLDBProvider.HSQLDBGlobalState state) { - super(state); - } - - @Override - protected ExpressionGenerator getGen() { - return expressionGenerator; - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getDatabaseTablesRandomSubsetNotEmpty(); - expressionGenerator = new HSQLDBExpressionGenerator(state) - .setColumns(targetTables.stream().flatMap(t -> t.getColumns().stream()).collect(Collectors.toList())); - initializeTernaryPredicateVariants(); - select = new HSQLDBSelect(); - select.setFetchColumns(generateFetchColumns()); - List tableList = targetTables.stream().map(t -> new HSQLDBTableReference(t)) - .collect(Collectors.toList()); - // List> joins = HSQLDBJoin.getJoins(tableList, state); - // select.setJoinList(joins.stream().collect(Collectors.toList())); - select.setFromList(tableList.stream().collect(Collectors.toList())); - select.setWhereClause(null); - - ComparatorHelper.getResultSetFirstColumnAsString(HSQLDBToStringVisitor.asString(select), errors, state); - - } - - List generateFetchColumns() { - List columns = new ArrayList<>(); - if (Randomly.getBoolean()) { - columns.add(new HSQLDBColumnReference(new HSQLDBSchema.HSQLDBColumn("*", null, null))); - } else { - columns = Randomly.nonEmptySubset(targetTables.stream().flatMap(t -> t.getColumns().stream()) - .map(c -> new HSQLDBColumnReference(c)).collect(Collectors.toList())); - } - return columns; - } -} diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java index 2b793383e..795c6656c 100644 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java @@ -1,41 +1,38 @@ package sqlancer.hsqldb.test; -import java.util.ArrayList; -import java.util.List; +import java.sql.SQLException; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.hsqldb.HSQLDBErrors; import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; -import sqlancer.hsqldb.HSQLDBToStringVisitor; +import sqlancer.hsqldb.HSQLDBSchema; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; +import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; +import sqlancer.hsqldb.ast.HSQLDBExpression; +import sqlancer.hsqldb.ast.HSQLDBJoin; +import sqlancer.hsqldb.ast.HSQLDBSelect; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; -public class HSQLDBQueryPartitioningWhereTester extends HSQLDBQueryPartitioningBase { +public class HSQLDBQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public HSQLDBQueryPartitioningWhereTester(HSQLDBGlobalState state) { - super(state); + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override - public void check() throws Exception { - super.check(); - String originalQueryString = HSQLDBToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(expressionGenerator.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = HSQLDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = HSQLDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = HSQLDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + public void check() throws SQLException { + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } } From 393c531124ca5a041838ad6a5bbddffb120481aa Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 16:47:01 +0800 Subject: [PATCH 549/856] [MySQL] Implement TLP-where expression generator interfaces --- src/sqlancer/mysql/ast/MySQLExpression.java | 5 ++- src/sqlancer/mysql/ast/MySQLJoin.java | 4 +- src/sqlancer/mysql/ast/MySQLSelect.java | 24 +++++++++- .../mysql/gen/MySQLExpressionGenerator.java | 44 ++++++++++++++++++- 4 files changed, 73 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/ast/MySQLExpression.java b/src/sqlancer/mysql/ast/MySQLExpression.java index 61a3b8aeb..1f3ae5bcb 100644 --- a/src/sqlancer/mysql/ast/MySQLExpression.java +++ b/src/sqlancer/mysql/ast/MySQLExpression.java @@ -1,6 +1,9 @@ package sqlancer.mysql.ast; -public interface MySQLExpression { +import sqlancer.common.ast.newast.Expression; +import sqlancer.mysql.MySQLSchema.MySQLColumn; + +public interface MySQLExpression extends Expression { default MySQLConstant getExpectedValue() { throw new AssertionError("PQS not supported for this operator"); diff --git a/src/sqlancer/mysql/ast/MySQLJoin.java b/src/sqlancer/mysql/ast/MySQLJoin.java index 13c21b7a0..c063b4f1b 100644 --- a/src/sqlancer/mysql/ast/MySQLJoin.java +++ b/src/sqlancer/mysql/ast/MySQLJoin.java @@ -5,12 +5,13 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.ast.newast.Join; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLExpressionGenerator; -public class MySQLJoin implements MySQLExpression { +public class MySQLJoin implements MySQLExpression, Join { public enum JoinType { NATURAL, INNER, STRAIGHT, LEFT, RIGHT, CROSS; @@ -44,6 +45,7 @@ public JoinType getType() { return type; } + @Override public void setOnClause(MySQLExpression onClause) { this.onClause = onClause; } diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index a50007417..7b9243c20 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -2,10 +2,16 @@ import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; -public class MySQLSelect extends SelectBase implements MySQLExpression { +public class MySQLSelect extends SelectBase + implements MySQLExpression, Select { private SelectType fromOptions = SelectType.ALL; private List modifiers = Collections.emptyList(); @@ -48,4 +54,20 @@ public MySQLText getHint() { return hint; } + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (MySQLExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (MySQLJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return MySQLVisitor.asString(this); + } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index c3d754f58..bddf5946f 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -2,14 +2,18 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; +import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; @@ -26,17 +30,22 @@ import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; +import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; +import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; -public class MySQLExpressionGenerator extends UntypedExpressionGenerator { +public class MySQLExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; + private List tables; public MySQLExpressionGenerator(MySQLGlobalState state) { this.state = state; @@ -198,4 +207,37 @@ public List generateOrderBys() { return newOrderBys; } + @Override + public TLPWhereGenerator setTablesAndColumns( + AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public MySQLExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public MySQLSelect generateSelect() { + return new MySQLSelect(); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new MySQLTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + return columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); + } } From 1e41016068b0b17883adfcb0490fc32f3370e60e Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 16:47:23 +0800 Subject: [PATCH 550/856] [MySQL] Use common TLP-where oracle --- src/sqlancer/mysql/MySQLErrors.java | 2 + .../oracle/MySQLQueryPartitioningBase.java | 61 ------------------- .../mysql/oracle/MySQLTLPWhereOracle.java | 59 +++++++++--------- 3 files changed, 32 insertions(+), 90 deletions(-) delete mode 100644 src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 9dcc0a5c3..13159e49e 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -32,6 +32,8 @@ public static List getExpressionRegexErrors() { // "00000000000000000000-0" } + errors.add(Pattern.compile("Unknown column '.*' in 'order clause'")); + return errors; } diff --git a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java b/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java deleted file mode 100644 index 7b7ba8b34..000000000 --- a/src/sqlancer/mysql/oracle/MySQLQueryPartitioningBase.java +++ /dev/null @@ -1,61 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mysql.MySQLErrors; -import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLSchema; -import sqlancer.mysql.MySQLSchema.MySQLTable; -import sqlancer.mysql.MySQLSchema.MySQLTables; -import sqlancer.mysql.ast.MySQLColumnReference; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLSelect; -import sqlancer.mysql.ast.MySQLTableReference; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -public abstract class MySQLQueryPartitioningBase extends - TernaryLogicPartitioningOracleBase implements TestOracle { - - MySQLSchema s; - MySQLTables targetTables; - MySQLExpressionGenerator gen; - MySQLSelect select; - - public MySQLQueryPartitioningBase(MySQLGlobalState state) { - super(state); - MySQLErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new MySQLExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new MySQLSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List tableList = tables.stream().map(t -> new MySQLTableReference(t)) - .collect(Collectors.toList()); - // List joins = MySQLJoin.getJoins(tableList, state); - select.setFromList(tableList); - select.setWhereClause(null); - // select.setJoins(joins); - } - - List generateFetchColumns() { - return Arrays.asList(MySQLColumnReference.create(targetTables.getColumns().get(0), null)); - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java index 3d4de3873..6d0fd1178 100644 --- a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java @@ -1,44 +1,45 @@ package sqlancer.mysql.oracle; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.MySQLSchema; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLJoin; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.gen.MySQLExpressionGenerator; -public class MySQLTLPWhereOracle extends MySQLQueryPartitioningBase { +public class MySQLTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public MySQLTLPWhereOracle(MySQLGlobalState state) { - super(state); + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = MySQLVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - if (Randomly.getBoolean()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = MySQLVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = MySQLVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = MySQLVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From 1fef4a3b4c313548b52cd573f4a42d4d31ff0be8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 17:48:46 +0800 Subject: [PATCH 551/856] [MySQL] Refactor environment check in test --- test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestMySQLPQS.java | 5 +---- test/sqlancer/dbms/TestMySQLTLP.java | 5 +---- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 8759ff5d5..92de68fe9 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -12,6 +12,7 @@ public class TestConfig { public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; + public static final String MYSQL_ENV = "MYSQL_AVAILABLE"; public static final String OCEANBASE_ENV = "OCEANBASE_AVAILABLE"; public static final String POSTGRES_ENV = "POSTGRES_AVAILABLE"; public static final String PRESTO_ENV = "PRESTO_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestMySQLPQS.java b/test/sqlancer/dbms/TestMySQLPQS.java index ba6a9c501..6f1b0786f 100644 --- a/test/sqlancer/dbms/TestMySQLPQS.java +++ b/test/sqlancer/dbms/TestMySQLPQS.java @@ -9,12 +9,9 @@ public class TestMySQLPQS { - String mysqlAvailable = System.getenv("MYSQL_AVAILABLE"); - boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); - @Test public void testPQS() { - assumeTrue(mysqlIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "4", "--random-string-generation", "ALPHANUMERIC", "--database-prefix", diff --git a/test/sqlancer/dbms/TestMySQLTLP.java b/test/sqlancer/dbms/TestMySQLTLP.java index 3d3e7d107..6254c58f3 100644 --- a/test/sqlancer/dbms/TestMySQLTLP.java +++ b/test/sqlancer/dbms/TestMySQLTLP.java @@ -9,12 +9,9 @@ public class TestMySQLTLP { - String mysqlAvailable = System.getenv("MYSQL_AVAILABLE"); - boolean mysqlIsAvailable = mysqlAvailable != null && mysqlAvailable.equalsIgnoreCase("true"); - @Test public void testMySQL() { - assumeTrue(mysqlIsAvailable); + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, From 045117e4a80ccbc9e0777b3c870a6dd2f614b5ad Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 18:07:40 +0800 Subject: [PATCH 552/856] [DataFusion] Implement TLP-where expression generator interface --- .../gen/DataFusionExpressionGenerator.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java index d56493fa2..520043fa1 100644 --- a/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionExpressionGenerator.java @@ -14,6 +14,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; @@ -35,7 +36,8 @@ public final class DataFusionExpressionGenerator extends TypedExpressionGenerator implements - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private List tables; private final DataFusionGlobalState globalState; @@ -234,8 +236,7 @@ public String getTextRepresentation() { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public DataFusionExpressionGenerator setTablesAndColumns(AbstractTables tables) { List randomTables = Randomly.nonEmptySubset(tables.getTables()); int maxSize = Randomly.fromOptions(1, 2, 3, 4); if (randomTables.size() > maxSize) { @@ -295,4 +296,10 @@ public String generateUnoptimizedQueryString(DataFusionSelect select, DataFusion return select.asString(); } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + List randomColumns = DataFusionTable.getRandomColumns(tables); + return randomColumns.stream().map((c) -> new DataFusionColumnReference(c)).collect(Collectors.toList()); + } } From 848f96c1bea253b5cbb0c5bceb5d9e130e4da5c9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Mon, 16 Sep 2024 18:07:58 +0800 Subject: [PATCH 553/856] [DataFusion] Use common TLP-where oracle --- .../datafusion/ast/DataFusionSelect.java | 39 ---------- .../test/DataFusionQueryPartitioningBase.java | 37 ---------- ...ataFusionQueryPartitioningWhereTester.java | 74 ++++++------------- 3 files changed, 23 insertions(+), 127 deletions(-) delete mode 100644 src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java diff --git a/src/sqlancer/datafusion/ast/DataFusionSelect.java b/src/sqlancer/datafusion/ast/DataFusionSelect.java index 10cf4859f..75bbb26ae 100644 --- a/src/sqlancer/datafusion/ast/DataFusionSelect.java +++ b/src/sqlancer/datafusion/ast/DataFusionSelect.java @@ -4,54 +4,15 @@ import java.util.Optional; import java.util.stream.Collectors; -import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; import sqlancer.common.ast.newast.Select; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionSchema; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; import sqlancer.datafusion.DataFusionSchema.DataFusionTable; import sqlancer.datafusion.DataFusionToStringVisitor; -import sqlancer.datafusion.gen.DataFusionExpressionGenerator; public class DataFusionSelect extends SelectBase implements DataFusionExpression, Select { public Optional fetchColumnsString = Optional.empty(); // When available, override `fetchColumns` in base - // class's `Node` representation (for display) - public DataFusionExpressionGenerator exprGen; - - // Construct a `DataFusionSelect` with random SELECT, FROM, WHERE - public static DataFusionSelect getRandomSelect(DataFusionGlobalState state) { - DataFusionSelect randomSelect = new DataFusionSelect(); - - // Randomly pick up to 4 tables to select from - DataFusionSchema schema = state.getSchema(); // schema of all tables - List allTables = schema.getDatabaseTables(); - List randomTables = Randomly.nonEmptySubset(allTables); - int maxSize = Randomly.fromOptions(1, 2, 3, 4); - if (randomTables.size() > maxSize) { - randomTables = randomTables.subList(0, maxSize); - } - - // Randomly choose some columns from `randomTables` - // And generate a random expression which might contain those columns - List randomColumns = DataFusionTable.getRandomColumns(randomTables); - randomSelect.exprGen = new DataFusionExpressionGenerator(state).setColumns(randomColumns); - DataFusionExpression whereExpr = randomSelect.exprGen - .generateExpression(DataFusionSchema.DataFusionDataType.BOOLEAN); - - // Constructing result - List randomTableNodes = randomTables.stream().map(t -> new DataFusionTableReference(t)) - .collect(Collectors.toList()); - List randomColumnNodes = randomColumns.stream() - .map((c) -> new DataFusionColumnReference(c)).collect(Collectors.toList()); - - randomSelect.setFetchColumns(randomColumnNodes); - randomSelect.setFromList(randomTableNodes); - randomSelect.setWhereClause(whereExpr); - - return randomSelect; - } /* * If set fetch columns with string It will override `fetchColumns` in base class when diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java deleted file mode 100644 index 067b07677..000000000 --- a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningBase.java +++ /dev/null @@ -1,37 +0,0 @@ -package sqlancer.datafusion.test; - -import java.sql.SQLException; - -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.ast.DataFusionExpression; -import sqlancer.datafusion.ast.DataFusionSelect; -import sqlancer.datafusion.gen.DataFusionExpressionGenerator; - -public class DataFusionQueryPartitioningBase - extends TernaryLogicPartitioningOracleBase - implements TestOracle { - DataFusionGlobalState state; - DataFusionExpressionGenerator gen; - DataFusionSelect select; - - public DataFusionQueryPartitioningBase(DataFusionGlobalState state) { - super(state); - this.state = state; - } - - @Override - public void check() throws SQLException { - select = DataFusionSelect.getRandomSelect(state); - gen = select.exprGen; - initializeTernaryPredicateVariants(); - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java index 3235c3d69..e0335a18f 100644 --- a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java +++ b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java @@ -1,67 +1,39 @@ package sqlancer.datafusion.test; -import static sqlancer.datafusion.DataFusionUtil.DataFusionLogger.DataFusionLogType.ERROR; - import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.datafusion.DataFusionErrors; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionToStringVisitor; -import sqlancer.datafusion.DataFusionUtil; +import sqlancer.datafusion.DataFusionSchema; +import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; +import sqlancer.datafusion.DataFusionSchema.DataFusionTable; +import sqlancer.datafusion.ast.DataFusionExpression; +import sqlancer.datafusion.ast.DataFusionJoin; import sqlancer.datafusion.ast.DataFusionSelect; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +public class DataFusionQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; -public class DataFusionQueryPartitioningWhereTester extends DataFusionQueryPartitioningBase { public DataFusionQueryPartitioningWhereTester(DataFusionGlobalState state) { - super(state); - DataFusionErrors.registerExpectedExecutionErrors(errors); + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) + .build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } - /* - * Query Partitioning - Where q: SELECT [expr1] FROM [expr2] qp1: SELECT [expr1] FROM [expr2] WHERE [expr3] qp2: - * SELECT [expr1] FROM [expr2] WHERE NOT [expr3] qp3: SELECT [expr1] FROM [expr2] WHERE [expr3] IS NULL - * - * Oracle check: q's result equals to union(qp1, qp2, qp3) - */ @Override public void check() throws SQLException { - // generate a random 'SELECT [expr1] FROM [expr2] WHERE [expr3] - super.check(); - DataFusionSelect randomSelect = select; - randomSelect.setWhereClause(null); - - // Construct q - String qString = DataFusionToStringVisitor.asString(randomSelect); - // Construct qp1, qp2, qp3 - randomSelect.setWhereClause(predicate); - String qp1String = DataFusionToStringVisitor.asString(randomSelect); - randomSelect.setWhereClause(negatedPredicate); - String qp2String = DataFusionToStringVisitor.asString(randomSelect); - randomSelect.setWhereClause(isNullPredicate); - String qp3String = DataFusionToStringVisitor.asString(randomSelect); - - try { - /* - * Run all queires - */ - List qResultSet = ComparatorHelper.getResultSetFirstColumnAsString(qString, errors, state); - List combinedString = new ArrayList<>(); - List qpResultSet = ComparatorHelper.getCombinedResultSet(qp1String, qp2String, qp3String, - combinedString, true, state, errors); - /* - * Query Partitioning-Where check - */ - ComparatorHelper.assumeResultSetsAreEqual(qResultSet, qpResultSet, qString, combinedString, state, - ComparatorHelper::canonicalizeResultValue); - } catch (AssertionError e) { - // Append more error message - String replay = DataFusionUtil.getReplay(state.getDatabaseName()); - String newMessage = e.getMessage() + "\n" + e.getCause() + "\n" + replay + "\n"; - state.dfLogger.appendToLog(ERROR, newMessage); + oracle.check(); + } - throw new AssertionError(newMessage); - } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From df714f1df7a119c2a479261aeb0ef37eee8122e3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:09:11 +0800 Subject: [PATCH 554/856] [OceanBase] Implement TLP-where expression generator interface --- .../oceanbase/gen/OceanBaseExpressionGenerator.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java index 3c17e5084..42b144c93 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseExpressionGenerator.java @@ -7,6 +7,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.oceanbase.OceanBaseGlobalState; @@ -40,8 +41,8 @@ import sqlancer.oceanbase.ast.OceanBaseUnaryPrefixOperation.OceanBaseUnaryPrefixOperator; public class OceanBaseExpressionGenerator extends UntypedExpressionGenerator - implements - NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private OceanBaseGlobalState state; private OceanBaseRowValue rowVal; @@ -225,8 +226,7 @@ public OceanBaseExpression isNull(OceanBaseExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public OceanBaseExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -360,4 +360,9 @@ private OceanBaseExpression getTrueExpr(OceanBaseExpression randomWhereCondition } return expr; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + return columns.stream().map(c -> new OceanBaseColumnReference(c, null)).collect(Collectors.toList()); + } } From 6709c8a8b30507630a4772c053a484ced79f865d Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:09:37 +0800 Subject: [PATCH 555/856] [OceanBase] Use common TLP-where oracle --- .../oceanbase/oracle/OceanBaseTLPBase.java | 63 ------------------- .../oracle/OceanBaseTLPWhereOracle.java | 59 ++++++++--------- 2 files changed, 30 insertions(+), 92 deletions(-) delete mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java deleted file mode 100644 index 73ff19bfb..000000000 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPBase.java +++ /dev/null @@ -1,63 +0,0 @@ -package sqlancer.oceanbase.oracle; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.oceanbase.OceanBaseErrors; -import sqlancer.oceanbase.OceanBaseGlobalState; -import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTables; -import sqlancer.oceanbase.ast.OceanBaseColumnReference; -import sqlancer.oceanbase.ast.OceanBaseExpression; -import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.ast.OceanBaseTableReference; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -import sqlancer.oceanbase.gen.OceanBaseHintGenerator; - -public abstract class OceanBaseTLPBase - extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - OceanBaseSchema s; - OceanBaseTables targetTables; - OceanBaseExpressionGenerator gen; - OceanBaseSelect select; - - public OceanBaseTLPBase(OceanBaseGlobalState state) { - super(state); - OceanBaseErrors.addExpressionErrors(errors); - errors.add("value is out of range"); - } - - @Override - public void check() throws SQLException { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - gen = new OceanBaseExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new OceanBaseSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - OceanBaseHintGenerator.generateHints(select, tables); - List tableList = tables.stream().map(t -> new OceanBaseTableReference(t)) - .collect(Collectors.toList()); - select.setFromList(tableList); - select.setWhereClause(null); - } - - List generateFetchColumns() { - return Arrays.asList(OceanBaseColumnReference.create(targetTables.getColumns().get(0), null)); - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java index 7c8c64e3c..08f5fef93 100644 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java +++ b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java @@ -1,44 +1,45 @@ package sqlancer.oceanbase.oracle; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; -import sqlancer.oceanbase.OceanBaseVisitor; +import sqlancer.oceanbase.OceanBaseSchema; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; +import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; +import sqlancer.oceanbase.ast.OceanBaseExpression; +import sqlancer.oceanbase.ast.OceanBaseJoin; +import sqlancer.oceanbase.ast.OceanBaseSelect; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; -public class OceanBaseTLPWhereOracle extends OceanBaseTLPBase { +public class OceanBaseTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public OceanBaseTLPWhereOracle(OceanBaseGlobalState state) { - super(state); + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).with("value is out of range").build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = OceanBaseVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - if (Randomly.getBoolean()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = OceanBaseVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = OceanBaseVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = OceanBaseVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From 276c2b7a1818ffe0f58c935125ca33031ec1ddb4 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:33:28 +0800 Subject: [PATCH 556/856] [ClickHouse] Implemenet TLP-where expression generator interface --- .../gen/ClickHouseExpressionGenerator.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java index b523703a3..4e04e91f4 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseExpressionGenerator.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; +import java.util.stream.IntStream; import com.clickhouse.client.ClickHouseDataType; @@ -31,12 +32,14 @@ import sqlancer.clickhouse.ast.ClickHouseUnaryPrefixOperation.ClickHouseUnaryPrefixOperator; import sqlancer.clickhouse.ast.constant.ClickHouseCreateConstant; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; public class ClickHouseExpressionGenerator extends TypedExpressionGenerator implements - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private final ClickHouseGlobalState globalState; public boolean allowAggregateFunctions; @@ -356,8 +359,7 @@ public ClickHouseExpression isNull(ClickHouseExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public ClickHouseExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.tables = tables.getTables(); this.columns = tables.getColumns(); return this; @@ -434,4 +436,15 @@ public String generateUnoptimizedQueryString(ClickHouseSelect select, ClickHouse select.setWhereClause(null); return "SELECT SUM(check <> 0) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new ClickHouseColumnReference(ClickHouseColumn.createDummy("*", null), null, null)); + } + List columnReferences = columns.stream() + .map(c -> c.asColumnReference(c.getTable().getName())).collect(Collectors.toList()); + return IntStream.range(0, 1 + Randomly.smallNumber()) + .mapToObj(i -> generateExpressionWithColumns(columnReferences, 5)).collect(Collectors.toList()); + } } From 33707784baed2cc113c29b56e306fb51f0716a11 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:33:45 +0800 Subject: [PATCH 557/856] [ClickHouse] Use common TLP-where oracle --- .../oracle/tlp/ClickHouseTLPWhereOracle.java | 69 +++++++++---------- 1 file changed, 32 insertions(+), 37 deletions(-) diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java index be2bd71fc..564db22f7 100644 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java @@ -1,50 +1,45 @@ package sqlancer.clickhouse.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.IntStream; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.clickhouse.ClickHouseErrors; -import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.clickhouse.ClickHouseVisitor; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.ClickHouseSchema; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; +import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; +import sqlancer.clickhouse.ast.ClickHouseExpression; +import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; +import sqlancer.clickhouse.ast.ClickHouseSelect; +import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; -public class ClickHouseTLPWhereOracle extends ClickHouseTLPBase { +public class ClickHouseTLPWhereOracle implements TestOracle { - public ClickHouseTLPWhereOracle(ClickHouseProvider.ClickHouseGlobalState state) { - super(state); - ClickHouseErrors.addExpectedExpressionErrors(errors); + private final TLPWhereOracle oracle; + + public ClickHouseTLPWhereOracle(ClickHouseGlobalState state) { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) + .build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) - .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); - } - String originalQueryString = ClickHouseVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(IntStream.range(0, 1 + Randomly.smallNumber()) - .mapToObj(i -> gen.generateExpressionWithColumns(columns, 5)).collect(Collectors.toList())); - } - - select.setWhereClause(predicate); - String firstQueryString = ClickHouseVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = ClickHouseVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = ClickHouseVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From 84278c8f17fad4390f0f2d999ca84b674b03aefc Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:48:33 +0800 Subject: [PATCH 558/856] [CockroachDB] Implement TLP-where expression generator interface --- .../gen/CockroachDBExpressionGenerator.java | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index d36e7673c..189be028d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -45,12 +45,14 @@ import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; public class CockroachDBExpressionGenerator extends TypedExpressionGenerator implements - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private List tables; private final CockroachDBGlobalState globalState; @@ -369,7 +371,7 @@ public CockroachDBExpression isNull(CockroachDBExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( + public CockroachDBExpressionGenerator setTablesAndColumns( AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -447,4 +449,13 @@ public String generateUnoptimizedQueryString(CockroachDBSelect select, Cockroach + " IS TRUE AS INT) as count FROM " + fromString + " " + joinList.stream().map(j -> CockroachDBVisitor.asString(j)).collect(Collectors.joining(", ")) + ")"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy || columns.size() == 0) { + return List.of(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new CockroachDBColumnReference(c)) + .collect(Collectors.toList()); + } } From e6d31efbd120f156a8c3055a108f2a65da98af62 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 11:48:58 +0800 Subject: [PATCH 559/856] [CockroachdB] Use common TLP-where oracle --- .../oracle/tlp/CockroachDBTLPWhereOracle.java | 60 ++++++++----------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java index 0fe935d18..5fc14c492 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java @@ -1,55 +1,45 @@ package sqlancer.cockroachdb.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.cockroachdb.CockroachDBSchema; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBNotOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; -import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; +import sqlancer.cockroachdb.ast.CockroachDBJoin; +import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; -public class CockroachDBTLPWhereOracle extends CockroachDBTLPBase { +public class CockroachDBTLPWhereOracle implements TestOracle { - private String generatedQueryString; + private final TLPWhereOracle oracle; public CockroachDBTLPWhereOracle(CockroachDBGlobalState state) { - super(state); - errors.add("GROUP BY term out of range"); + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with("GROUP BY term out of range").build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - String originalQueryString = CockroachDBVisitor.asString(select); - generatedQueryString = originalQueryString; - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean allowOrderBy = Randomly.getBoolean(); - if (allowOrderBy) { - select.setOrderByClauses(gen.getOrderingTerms()); - } - CockroachDBExpression predicate = gen.generateExpression(CockroachDBDataType.BOOL.get()); - select.setWhereClause(predicate); - String firstQueryString = CockroachDBVisitor.asString(select); - select.setWhereClause(new CockroachDBNotOperation(predicate)); - String secondQueryString = CockroachDBVisitor.asString(select); - select.setWhereClause(new CockroachDBUnaryPostfixOperation(predicate, CockroachDBUnaryPostfixOperator.IS_NULL)); - String thirdQueryString = CockroachDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !allowOrderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + oracle.check(); } @Override public String getLastQueryString() { - return generatedQueryString; + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From 6f0a07ed46a2302ee1a43b859c9d976a9e66d116 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:15:57 +0800 Subject: [PATCH 560/856] [Databend] Implement TLP-where expression generator interface --- .../gen/DatabendNewExpressionGenerator.java | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java index a3b7e996e..20901bc1b 100644 --- a/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java +++ b/src/sqlancer/databend/gen/DatabendNewExpressionGenerator.java @@ -10,6 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.databend.DatabendBugs; @@ -48,7 +49,8 @@ public class DatabendNewExpressionGenerator extends TypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private final DatabendGlobalState globalState; private List tables; @@ -355,8 +357,7 @@ public DatabendExpression generateHavingClause() { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public DatabendNewExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -421,4 +422,11 @@ public String generateUnoptimizedQueryString(DatabendSelect select, DatabendExpr return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new DatabendColumnReference(new DatabendColumn("*", null, false, false))); + } + return columns.stream().map(c -> new DatabendColumnReference(c)).collect(Collectors.toList()); + } } From 98d40769b4932e9c9284ee48afa0307271e336b0 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:16:18 +0800 Subject: [PATCH 561/856] [Databend] Use common TLP-where oracle --- .../DatabendQueryPartitioningWhereTester.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java index 6ea6f5e57..b6520e7f2 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java @@ -1,45 +1,45 @@ package sqlancer.databend.test.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendToStringVisitor; +import sqlancer.databend.DatabendSchema; +import sqlancer.databend.DatabendSchema.DatabendColumn; +import sqlancer.databend.DatabendSchema.DatabendTable; +import sqlancer.databend.ast.DatabendExpression; +import sqlancer.databend.ast.DatabendJoin; +import sqlancer.databend.ast.DatabendSelect; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; -public class DatabendQueryPartitioningWhereTester extends DatabendQueryPartitioningBase { +public class DatabendQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public DatabendQueryPartitioningWhereTester(DatabendGlobalState state) { - super(state); - DatabendErrors.addGroupByErrors(errors); + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with(DatabendErrors.getGroupByErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = DatabendToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - // boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - boolean orderBy = false; - // if (orderBy) { //TODO 待开启 - // select.setOrderByClauses(gen.generateOrderBys()); - // } - select.setWhereClause(predicate); - String firstQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = DatabendToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = DatabendToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From eadad7b6784f5ac5aa66ef17c36ef91629c4fcf9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:25:14 +0800 Subject: [PATCH 562/856] [Doris] Implement TLP-where expression generator interfaces --- .../doris/gen/DorisNewExpressionGenerator.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java index 7cea29f31..bddc6cc5a 100644 --- a/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java +++ b/src/sqlancer/doris/gen/DorisNewExpressionGenerator.java @@ -11,6 +11,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.doris.DorisBugs; @@ -50,7 +51,8 @@ import sqlancer.doris.visitor.DorisToStringVisitor; public class DorisNewExpressionGenerator extends TypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private final DorisGlobalState globalState; private List tables; @@ -463,8 +465,7 @@ public void setAllowAggregateFunctions(boolean allowAggregateFunctions) { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public DorisNewExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -532,4 +533,13 @@ public String generateUnoptimizedQueryString(DorisSelect select, DorisExpression return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new DorisColumnReference(new DorisColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new DorisColumnReference(c)) + .collect(Collectors.toList()); + } } From 37c4963d340a58df013d87504ee0f3d2f44daace Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:25:35 +0800 Subject: [PATCH 563/856] [Doris] Use common TLP-where oracle --- .../DorisQueryPartitioningWhereTester.java | 62 +++++++++---------- 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java index 5a593a573..32b171af0 100644 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java @@ -1,51 +1,45 @@ package sqlancer.doris.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.ast.DorisConstant; +import sqlancer.doris.DorisSchema; +import sqlancer.doris.DorisSchema.DorisColumn; +import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.visitor.DorisToStringVisitor; +import sqlancer.doris.ast.DorisJoin; +import sqlancer.doris.ast.DorisSelect; +import sqlancer.doris.gen.DorisNewExpressionGenerator; -public class DorisQueryPartitioningWhereTester extends DorisQueryPartitioningBase { +public class DorisQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public DorisQueryPartitioningWhereTester(DorisGlobalState state) { - super(state); - DorisErrors.addExpressionErrors(errors); - DorisErrors.addInsertErrors(errors); + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with(DorisErrors.getExpressionErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = DorisToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - List constants = new ArrayList<>(); - constants.add( - new DorisConstant.DorisIntConstant(Randomly.smallNumber() % select.getFetchColumns().size() + 1)); - select.setOrderByClauses(constants); - } - select.setWhereClause(predicate); - String firstQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = DorisToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = DorisToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From 8a32ad7ed2116113953fcbf8caebd338183a4c6f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:35:20 +0800 Subject: [PATCH 564/856] [DuckDB] Implement TLP-where expression generator interface --- .../duckdb/gen/DuckDBExpressionGenerator.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 3f6abc37f..278685a46 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -11,6 +11,7 @@ import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -35,7 +36,8 @@ import sqlancer.duckdb.ast.DuckDBTernary; public final class DuckDBExpressionGenerator extends UntypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private final DuckDBGlobalState globalState; private List tables; @@ -449,8 +451,7 @@ public DuckDBExpression isNull(DuckDBExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public DuckDBExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -513,4 +514,13 @@ public String generateUnoptimizedQueryString(DuckDBSelect select, DuckDBExpressi return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (Randomly.getBoolean()) { + return List.of(new DuckDBColumnReference(new DuckDBColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new DuckDBColumnReference(c)) + .collect(Collectors.toList()); + } } From 85281e2b1ff840446d89571c17ce0ef3186450da Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:36:03 +0800 Subject: [PATCH 565/856] [DuckDB] Use common TLP-where oracle --- .../DuckDBQueryPartitioningWhereTester.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java index 7e20e605c..d4ffa74c8 100644 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java @@ -1,45 +1,45 @@ package sqlancer.duckdb.test; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.DuckDBSchema; +import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; +import sqlancer.duckdb.DuckDBSchema.DuckDBTable; +import sqlancer.duckdb.ast.DuckDBExpression; +import sqlancer.duckdb.ast.DuckDBJoin; +import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator; -public class DuckDBQueryPartitioningWhereTester extends DuckDBQueryPartitioningBase { +public class DuckDBQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public DuckDBQueryPartitioningWhereTester(DuckDBGlobalState state) { - super(state); - DuckDBErrors.addGroupByErrors(errors); + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .with(DuckDBErrors.getGroupByErrors()).withRegex(DuckDBErrors.getExpressionErrorsRegex()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = DuckDBToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = DuckDBToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = DuckDBToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = DuckDBToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From e486f85fbc6a7131d1eca35f2df3c69a8b8da869 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:51:20 +0800 Subject: [PATCH 566/856] [Materialize] Implement TLP-where expression generator interface --- .../gen/MaterializeExpressionGenerator.java | 24 +++++++++++++++---- .../gen/MaterializeRandomQueryGenerator.java | 2 +- ...aterializePivotedQuerySynthesisOracle.java | 2 +- .../tlp/MaterializeTLPAggregateOracle.java | 2 +- .../oracle/tlp/MaterializeTLPBase.java | 2 +- .../tlp/MaterializeTLPHavingOracle.java | 2 +- 6 files changed, 25 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java index f3af50038..d61e452f8 100644 --- a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -12,6 +12,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.materialize.MaterializeCompoundDataType; import sqlancer.materialize.MaterializeGlobalState; @@ -59,7 +60,8 @@ import sqlancer.materialize.oracle.tlp.MaterializeTLPBase; public class MaterializeExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private final int maxDepth; @@ -103,7 +105,8 @@ public MaterializeExpression generateExpression(int depth) { return generateExpression(depth, MaterializeDataType.getRandomType()); } - public List generateOrderBy() { + @Override + public List generateOrderBys() { List orderBys = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber(); i++) { orderBys.add(new MaterializeOrderByTerm(MaterializeColumnValue.create(Randomly.fromList(columns), null), @@ -530,7 +533,7 @@ public MaterializeExpression isNull(MaterializeExpression expr) { } @Override - public NoRECGenerator setTablesAndColumns( + public MaterializeExpressionGenerator setTablesAndColumns( AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -593,7 +596,7 @@ public String generateOptimizedQueryString(MaterializeSelect select, Materialize MaterializeColumnValue allColumns = new MaterializeColumnValue(Randomly.fromList(columns), null); select.setFetchColumns(List.of(allColumns)); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(generateOrderBy()); + select.setOrderByClauses(generateOrderBys()); } select.setSelectType(SelectType.ALL); } @@ -612,4 +615,17 @@ public String generateUnoptimizedQueryString(MaterializeSelect select, Materiali return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy) { + return List.of(new MaterializeColumnValue(MaterializeColumn.createDummy("*"), null)); + } + List fetchColumns = new ArrayList<>(); + List targetColumns = Randomly.nonEmptySubset(columns); + for (MaterializeColumn c : targetColumns) { + fetchColumns.add(new MaterializeColumnValue(c, null)); + } + return fetchColumns; + } } diff --git a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java index c8b1441fc..05d908e05 100644 --- a/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeRandomQueryGenerator.java @@ -46,7 +46,7 @@ public static MaterializeSelect createRandomQuery(int nrColumns, MaterializeGlob } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java index f4b2a1cc6..50dff2b49 100644 --- a/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java +++ b/src/sqlancer/materialize/oracle/MaterializePivotedQuerySynthesisOracle.java @@ -63,7 +63,7 @@ public SQLQueryAdapter getRectifiedQuery() throws SQLException { selectStatement.setOffsetClause(offsetClause); } List orderBy = new MaterializeExpressionGenerator(globalState).setColumns(columns) - .generateOrderBy(); + .generateOrderBys(); selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(MaterializeVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java index e8154bb9a..6bbd7d795 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPAggregateOracle.java @@ -63,7 +63,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = MaterializeVisitor.asString(select); generatedQueryString = originalQuery; diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java index 36711997a..faf136c21 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPBase.java @@ -127,7 +127,7 @@ public static MaterializeSubquery createSubquery(MaterializeGlobalState globalSt select.setWhereClause(gen.generateExpression(0, MaterializeDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(MaterializeConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java index e4328ce40..deefb4fb9 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPHavingOracle.java @@ -49,7 +49,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setHavingClause(predicate); String firstQueryString = MaterializeVisitor.asString(select); From a736f3233a4969c530378f4530433040078dee96 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:51:47 +0800 Subject: [PATCH 567/856] [Materialize] Use common TLP-where oracle --- .../oracle/tlp/MaterializeTLPWhereOracle.java | 59 ++++++++----------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java index 6cec373a9..cd14bfb75 100644 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java +++ b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java @@ -1,52 +1,45 @@ package sqlancer.materialize.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.materialize.MaterializeGlobalState; -import sqlancer.materialize.MaterializeVisitor; +import sqlancer.materialize.MaterializeSchema; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; +import sqlancer.materialize.MaterializeSchema.MaterializeTable; +import sqlancer.materialize.ast.MaterializeExpression; +import sqlancer.materialize.ast.MaterializeJoin; +import sqlancer.materialize.ast.MaterializeSelect; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; -public class MaterializeTLPWhereOracle extends MaterializeTLPBase { - private String generatedQueryString; +public class MaterializeTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public MaterializeTLPWhereOracle(MaterializeGlobalState state) { - super(state); + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) + .with(MaterializeCommon.getCommonFetchErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - whereCheck(); + oracle.check(); } - protected void whereCheck() throws SQLException { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - String originalQueryString = MaterializeVisitor.asString(select); - generatedQueryString = originalQueryString; - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = MaterializeVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = MaterializeVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = MaterializeVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } @Override - public String getLastQueryString() { - return generatedQueryString; + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From 2dd43a25e6c600401bee04a0469f6434a5724c0f Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:58:47 +0800 Subject: [PATCH 568/856] [Presto] Implement TLP-where expression generator interfaces --- .../gen/PrestoTypedExpressionGenerator.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java index 42e3b8691..4a788f01b 100644 --- a/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTypedExpressionGenerator.java @@ -10,6 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; @@ -46,7 +47,8 @@ public final class PrestoTypedExpressionGenerator extends TypedExpressionGenerator - implements NoRECGenerator { + implements NoRECGenerator, + TLPWhereGenerator { private final Randomly randomly; private final PrestoGlobalState globalState; @@ -803,8 +805,7 @@ private enum Expression { } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public PrestoTypedExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -874,4 +875,13 @@ public String generateUnoptimizedQueryString(PrestoSelect select, PrestoExpressi select.setWhereClause(null); return "SELECT SUM(count) FROM (" + PrestoToStringVisitor.asString(select) + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (Randomly.getBoolean()) { + return List.of(new PrestoColumnReference(new PrestoColumn("*", null, false, false))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new PrestoColumnReference(c)) + .collect(Collectors.toList()); + } } From 3d3bc2c128b29e7fd0031ecb32d906a4f96f61d3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 17 Sep 2024 14:59:04 +0800 Subject: [PATCH 569/856] [Presto] Use common TLP-where oracle --- .../PrestoQueryPartitioningWhereTester.java | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java index cfff823c5..1fb2b7018 100644 --- a/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java +++ b/src/sqlancer/presto/test/PrestoQueryPartitioningWhereTester.java @@ -1,46 +1,45 @@ package sqlancer.presto.test; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; -import sqlancer.presto.PrestoToStringVisitor; +import sqlancer.presto.PrestoSchema; +import sqlancer.presto.PrestoSchema.PrestoColumn; +import sqlancer.presto.PrestoSchema.PrestoTable; +import sqlancer.presto.ast.PrestoExpression; +import sqlancer.presto.ast.PrestoJoin; +import sqlancer.presto.ast.PrestoSelect; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; -public class PrestoQueryPartitioningWhereTester extends PrestoQueryPartitioningBase { +public class PrestoQueryPartitioningWhereTester implements TestOracle { + + private final TLPWhereOracle oracle; public PrestoQueryPartitioningWhereTester(PrestoGlobalState state) { - super(state); - PrestoErrors.addGroupByErrors(errors); - PrestoErrors.addExpressionErrors(errors); + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) + .with(PrestoErrors.getGroupByErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - select.setWhereClause(null); - String originalQueryString = PrestoToStringVisitor.asString(select); - - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - select.setWhereClause(predicate); - String firstQueryString = PrestoToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = PrestoToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = PrestoToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, PrestoQueryPartitioningBase::canonicalizeResultValue); + oracle.check(); } + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); + } + + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); + } } From 9b509a744bebf227d38e9ae35580d7e671630a58 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 11 Sep 2024 14:23:15 +0800 Subject: [PATCH 570/856] [Citus] Use common TLP Where oracle --- src/sqlancer/citus/gen/CitusCommon.java | 2 +- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 66 +++++++------------ 2 files changed, 24 insertions(+), 44 deletions(-) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 1ca2af138..317dfc0f1 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -54,6 +54,7 @@ public static List getCitusErrors() { errors.add("unlogged columnar tables are not supported"); errors.add("UPDATE and CTID scans not supported for ColumnarScan"); errors.add("indexes not supported for columnar tables"); + errors.add("invalid byte sequence for encoding \"UTF8\": 0x00"); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { @@ -78,7 +79,6 @@ public static List getCitusErrors() { if (CitusBugs.bug4079) { errors.add("aggregate function calls cannot be nested"); } - return errors; } diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java index dbfc26e4b..9d1b13f41 100644 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java @@ -1,62 +1,42 @@ package sqlancer.citus.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; import sqlancer.citus.CitusGlobalState; import sqlancer.citus.gen.CitusCommon; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresVisitor; -import sqlancer.postgres.oracle.tlp.PostgresTLPBase; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresColumn; +import sqlancer.postgres.PostgresSchema.PostgresTable; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresSelect; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; -public class CitusTLPWhereOracle extends PostgresTLPBase { +public class CitusTLPWhereOracle implements TestOracle { - private final CitusTLPBase citusTLPBase; + private final TLPWhereOracle oracle; public CitusTLPWhereOracle(CitusGlobalState state) { - super(state); - CitusCommon.addCitusErrors(errors); - citusTLPBase = new CitusTLPBase(state); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .with(CitusCommon.getCitusErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - state.setAllowedFunctionTypes(Arrays.asList(PostgresGlobalState.IMMUTABLE)); - citusTLPBase.check(); - s = citusTLPBase.getSchema(); - targetTables = citusTLPBase.getTargetTables(); - gen = citusTLPBase.getGenerator(); - select = citusTLPBase.getSelect(); - predicate = citusTLPBase.getPredicate(); - negatedPredicate = citusTLPBase.getNegatedPredicate(); - isNullPredicate = citusTLPBase.getIsNullPredicate(); - whereCheck(); - state.setDefaultAllowedFunctionTypes(); + oracle.check(); } - void whereCheck() throws SQLException { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - String originalQueryString = PostgresVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = PostgresVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = PostgresVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = PostgresVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } } From c234b11a15d9048348e31b7951c81217560e3258 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 21 Sep 2024 16:09:32 +0800 Subject: [PATCH 571/856] [Citus] Update expected errors --- src/sqlancer/citus/gen/CitusCommon.java | 5 +++++ src/sqlancer/postgres/gen/PostgresTruncateGenerator.java | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 317dfc0f1..7a9f9c659 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -55,6 +55,11 @@ public static List getCitusErrors() { errors.add("UPDATE and CTID scans not supported for ColumnarScan"); errors.add("indexes not supported for columnar tables"); errors.add("invalid byte sequence for encoding \"UTF8\": 0x00"); + errors.add("columnar_tuple_insert_speculative not implemented"); + errors.add("row field count is 1, expected 2"); + errors.add("incorrect binary data format"); + errors.add("invalid sign in external \"numeric\" value"); + errors.add("Foreign keys and AFTER ROW triggers are not supported for columnar tables"); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { diff --git a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java index 197668ac5..0745d1cce 100644 --- a/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTruncateGenerator.java @@ -33,8 +33,9 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { sb.append(" "); sb.append(Randomly.fromOptions("CASCADE", "RESTRICT")); } - return new SQLQueryAdapter(sb.toString(), ExpectedErrors - .from("cannot truncate a table referenced in a foreign key constraint", "is not a table")); + return new SQLQueryAdapter(sb.toString(), + ExpectedErrors.from("cannot truncate a table referenced in a foreign key constraint", "is not a table", + "is not distributed")); } } From 3131e3f34d2a27f1a63b5f9213c73b98cbbc40b3 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Sat, 21 Sep 2024 16:05:27 +0800 Subject: [PATCH 572/856] [Presto] Update expected errors --- src/sqlancer/presto/PrestoErrors.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 0e000119e..2296223d6 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -41,6 +41,7 @@ public static List getExpressionErrors() { // TODO: check errors.add("io.airlift.slice.Slice cannot be cast to java.lang.Number"); + errors.add("class io.airlift.slice.Slice cannot be cast to class java.lang.Number"); if (PrestoBugs.bug23324) { errors.add("Cannot cast java.lang.Long to io.airlift.slice.Slice"); } @@ -74,6 +75,11 @@ public static List getExpressionErrors() { errors.add("All IN list values must be the same type"); errors.add("All CASE results must be the same type"); errors.add("Mismatched types"); + errors.add("CASE operand type does not match WHEN clause operand type"); + errors.add("Subquery result type must be orderable"); + errors.add("Escape character must be followed by '%', '_' or the escape character itself"); + errors.add("Types are not comparable with NULLIF"); + errors.add("not of the same type"); if (PrestoBugs.bug23613) { errors.add("at index 1"); From 0e514c14e5c8da12667fb318f45faa57c971a511 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 25 Sep 2024 11:47:17 +0800 Subject: [PATCH 573/856] Refactor test oracle factory --- src/sqlancer/citus/CitusOptions.java | 48 -------- src/sqlancer/citus/CitusOracleFactory.java | 70 +++++++++++ .../citus/oracle/CitusNoRECOracle.java | 46 -------- .../citus/oracle/tlp/CitusTLPWhereOracle.java | 42 ------- .../clickhouse/ClickHouseOptions.java | 51 -------- .../clickhouse/ClickHouseOracleFactory.java | 62 ++++++++++ .../oracle/norec/ClickHouseNoRECOracle.java | 45 -------- .../oracle/tlp/ClickHouseTLPWhereOracle.java | 45 -------- src/sqlancer/cnosdb/CnosDBOptions.java | 36 ------ src/sqlancer/cnosdb/CnosDBOracleFactory.java | 39 +++++++ .../cockroachdb/CockroachDBOptions.java | 87 -------------- .../cockroachdb/CockroachDBOracleFactory.java | 109 ++++++++++++++++++ .../cockroachdb/CockroachDBProvider.java | 1 - .../CockroachDBRandomQuerySynthesizer.java | 4 +- .../oracle/CockroachDBNoRECOracle.java | 69 ----------- .../tlp/CockroachDBTLPAggregateOracle.java | 3 +- .../oracle/tlp/CockroachDBTLPBase.java | 21 +++- .../oracle/tlp/CockroachDBTLPWhereOracle.java | 45 -------- src/sqlancer/databend/DatabendOptions.java | 76 ------------ .../databend/DatabendOracleFactory.java | 93 +++++++++++++++ .../databend/test/DatabendNoRECOracle.java | 45 -------- .../DatabendQueryPartitioningWhereTester.java | 45 -------- .../datafusion/DataFusionOptions.java | 22 ---- .../datafusion/DataFusionOracleFactory.java | 34 ++++++ .../test/DataFusionNoRECOracle.java | 44 ------- ...ataFusionQueryPartitioningWhereTester.java | 39 ------- src/sqlancer/doris/DorisOptions.java | 88 -------------- src/sqlancer/doris/DorisOracleFactory.java | 108 +++++++++++++++++ .../doris/oracle/DorisNoRECOracle.java | 44 ------- .../DorisQueryPartitioningWhereTester.java | 45 -------- src/sqlancer/duckdb/DuckDBOptions.java | 69 ----------- src/sqlancer/duckdb/DuckDBOracleFactory.java | 86 ++++++++++++++ .../duckdb/test/DuckDBNoRECOracle.java | 45 -------- .../DuckDBQueryPartitioningWhereTester.java | 45 -------- src/sqlancer/h2/H2Options.java | 18 --- src/sqlancer/h2/H2OracleFactory.java | 23 ++++ .../h2/H2QueryPartitioningWhereTester.java | 35 ------ src/sqlancer/hsqldb/HSQLDBOptions.java | 27 +---- src/sqlancer/hsqldb/HSQLDBOracleFactory.java | 32 +++++ .../hsqldb/test/HSQLDBNoRECOracle.java | 43 ------- .../HSQLDBQueryPartitioningWhereTester.java | 38 ------ src/sqlancer/mariadb/MariaDBOptions.java | 25 ---- .../mariadb/MariaDBOracleFactory.java | 36 ++++++ .../mariadb/oracle/MariaDBNoRECOracle.java | 48 -------- .../materialize/MaterializeOptions.java | 50 -------- .../materialize/MaterializeOracleFactory.java | 70 +++++++++++ .../materialize/MaterializeProvider.java | 1 - .../oracle/MaterializeNoRECOracle.java | 45 -------- .../oracle/tlp/MaterializeTLPWhereOracle.java | 45 -------- src/sqlancer/mysql/MySQLGlobalState.java | 1 - src/sqlancer/mysql/MySQLOptions.java | 58 ---------- src/sqlancer/mysql/MySQLOracleFactory.java | 64 ++++++++++ src/sqlancer/mysql/MySQLProvider.java | 1 - .../mysql/oracle/MySQLTLPWhereOracle.java | 45 -------- .../oceanbase/OceanBaseGlobalState.java | 1 - src/sqlancer/oceanbase/OceanBaseOptions.java | 35 ------ .../oceanbase/OceanBaseOracleFactory.java | 50 ++++++++ .../oracle/OceanBaseNoRECOracle.java | 49 -------- .../oracle/OceanBaseTLPWhereOracle.java | 45 -------- src/sqlancer/postgres/PostgresOptions.java | 70 ----------- .../postgres/PostgresOracleFactory.java | 91 +++++++++++++++ src/sqlancer/postgres/PostgresProvider.java | 1 - .../postgres/oracle/PostgresNoRECOracle.java | 79 ------------- .../postgres/oracle/tlp/PostgresTLPBase.java | 25 +++- .../oracle/tlp/PostgresTLPWhereOracle.java | 40 ------- src/sqlancer/presto/PrestoOptions.java | 66 +---------- src/sqlancer/presto/PrestoOracleFactory.java | 73 ++++++++++++ .../presto/test/PrestoNoRECOracle.java | 44 ------- src/sqlancer/questdb/QuestDBOptions.java | 16 --- .../questdb/QuestDBOracleFactory.java | 18 +++ src/sqlancer/sqlite3/SQLite3Options.java | 88 -------------- .../sqlite3/SQLite3OracleFactory.java | 101 ++++++++++++++++ src/sqlancer/sqlite3/SQLite3Provider.java | 1 - .../sqlite3/gen/SQLite3ColumnBuilder.java | 2 +- .../gen/ddl/SQLite3TableGenerator.java | 2 +- .../sqlite3/gen/ddl/SQLite3ViewGenerator.java | 2 +- .../sqlite3/oracle/SQLite3NoRECOracle.java | 48 -------- .../oracle/tlp/SQLite3TLPWhereOracle.java | 37 ------ src/sqlancer/tidb/TiDBOptions.java | 53 --------- src/sqlancer/tidb/TiDBOracleFactory.java | 64 ++++++++++ src/sqlancer/tidb/TiDBProvider.java | 1 - .../tidb/oracle/TiDBTLPWhereOracle.java | 44 ------- src/sqlancer/yugabyte/ycql/YCQLOptions.java | 16 --- .../yugabyte/ycql/YCQLOracleFactory.java | 18 +++ src/sqlancer/yugabyte/ysql/YSQLOptions.java | 63 ---------- .../yugabyte/ysql/YSQLOracleFactory.java | 72 ++++++++++++ .../yugabyte/ysql/oracle/YSQLNoRECOracle.java | 44 ------- 87 files changed, 1363 insertions(+), 2452 deletions(-) create mode 100644 src/sqlancer/citus/CitusOracleFactory.java delete mode 100644 src/sqlancer/citus/oracle/CitusNoRECOracle.java delete mode 100644 src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java create mode 100644 src/sqlancer/clickhouse/ClickHouseOracleFactory.java delete mode 100644 src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java delete mode 100644 src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java create mode 100644 src/sqlancer/cnosdb/CnosDBOracleFactory.java create mode 100644 src/sqlancer/cockroachdb/CockroachDBOracleFactory.java delete mode 100644 src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java delete mode 100644 src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java create mode 100644 src/sqlancer/databend/DatabendOracleFactory.java delete mode 100644 src/sqlancer/databend/test/DatabendNoRECOracle.java delete mode 100644 src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/datafusion/DataFusionOracleFactory.java delete mode 100644 src/sqlancer/datafusion/test/DataFusionNoRECOracle.java delete mode 100644 src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/doris/DorisOracleFactory.java delete mode 100644 src/sqlancer/doris/oracle/DorisNoRECOracle.java delete mode 100644 src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/duckdb/DuckDBOracleFactory.java delete mode 100644 src/sqlancer/duckdb/test/DuckDBNoRECOracle.java delete mode 100644 src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/h2/H2OracleFactory.java delete mode 100644 src/sqlancer/h2/H2QueryPartitioningWhereTester.java create mode 100644 src/sqlancer/hsqldb/HSQLDBOracleFactory.java delete mode 100644 src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java delete mode 100644 src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java create mode 100644 src/sqlancer/mariadb/MariaDBOracleFactory.java delete mode 100644 src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java create mode 100644 src/sqlancer/materialize/MaterializeOracleFactory.java delete mode 100644 src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java delete mode 100644 src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java create mode 100644 src/sqlancer/mysql/MySQLOracleFactory.java delete mode 100644 src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java create mode 100644 src/sqlancer/oceanbase/OceanBaseOracleFactory.java delete mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java delete mode 100644 src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java create mode 100644 src/sqlancer/postgres/PostgresOracleFactory.java delete mode 100644 src/sqlancer/postgres/oracle/PostgresNoRECOracle.java delete mode 100644 src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java create mode 100644 src/sqlancer/presto/PrestoOracleFactory.java delete mode 100644 src/sqlancer/presto/test/PrestoNoRECOracle.java create mode 100644 src/sqlancer/questdb/QuestDBOracleFactory.java create mode 100644 src/sqlancer/sqlite3/SQLite3OracleFactory.java delete mode 100644 src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java delete mode 100644 src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java create mode 100644 src/sqlancer/tidb/TiDBOracleFactory.java delete mode 100644 src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java create mode 100644 src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java create mode 100644 src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java delete mode 100644 src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java diff --git a/src/sqlancer/citus/CitusOptions.java b/src/sqlancer/citus/CitusOptions.java index 67b14c304..f2d1b0abc 100644 --- a/src/sqlancer/citus/CitusOptions.java +++ b/src/sqlancer/citus/CitusOptions.java @@ -1,22 +1,11 @@ package sqlancer.citus; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameter; -import sqlancer.OracleFactory; -import sqlancer.citus.oracle.CitusNoRECOracle; -import sqlancer.citus.oracle.tlp.CitusTLPAggregateOracle; -import sqlancer.citus.oracle.tlp.CitusTLPHavingOracle; -import sqlancer.citus.oracle.tlp.CitusTLPWhereOracle; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresOptions; -import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; public class CitusOptions extends PostgresOptions { @@ -26,41 +15,4 @@ public class CitusOptions extends PostgresOptions { @Parameter(names = "--citusoracle", description = "Specifies which test oracle should be used for Citus extension to PostgreSQL") public List citusOracle = Arrays.asList(CitusOracleFactory.QUERY_PARTITIONING); - public enum CitusOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; - return new CitusNoRECOracle(citusGlobalState); - } - }, - PQS { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresPivotedQuerySynthesisOracle(globalState); - } - }, - HAVING { - - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; - return new CitusTLPHavingOracle(citusGlobalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; - List> oracles = new ArrayList<>(); - oracles.add(new CitusTLPWhereOracle(citusGlobalState)); - oracles.add(new CitusTLPHavingOracle(citusGlobalState)); - oracles.add(new CitusTLPAggregateOracle(citusGlobalState)); - return new CompositeTestOracle(oracles, globalState); - } - }; - - } - } diff --git a/src/sqlancer/citus/CitusOracleFactory.java b/src/sqlancer/citus/CitusOracleFactory.java new file mode 100644 index 000000000..b6e0a7f3e --- /dev/null +++ b/src/sqlancer/citus/CitusOracleFactory.java @@ -0,0 +1,70 @@ +package sqlancer.citus; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.citus.gen.CitusCommon; +import sqlancer.citus.oracle.tlp.CitusTLPAggregateOracle; +import sqlancer.citus.oracle.tlp.CitusTLPHavingOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; + +public enum CitusOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .with(CitusCommon.getCitusErrors().toArray(new String[0])).build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + }, + WHERE { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()).with(CitusCommon.getCitusErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + HAVING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; + return new CitusTLPHavingOracle(citusGlobalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + CitusGlobalState citusGlobalState = (CitusGlobalState) globalState; + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(citusGlobalState)); + oracles.add(HAVING.create(citusGlobalState)); + oracles.add(new CitusTLPAggregateOracle(citusGlobalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/citus/oracle/CitusNoRECOracle.java b/src/sqlancer/citus/oracle/CitusNoRECOracle.java deleted file mode 100644 index ffc7c13fb..000000000 --- a/src/sqlancer/citus/oracle/CitusNoRECOracle.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.citus.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.citus.gen.CitusCommon; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; - -public class CitusNoRECOracle implements TestOracle { - private final NoRECOracle oracle; - - public CitusNoRECOracle(PostgresGlobalState globalState) { - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) - .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) - .with(CitusCommon.getCitusErrors().toArray(new String[0])).build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - -} diff --git a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java b/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java deleted file mode 100644 index 9d1b13f41..000000000 --- a/src/sqlancer/citus/oracle/tlp/CitusTLPWhereOracle.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.citus.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.citus.CitusGlobalState; -import sqlancer.citus.gen.CitusCommon; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; - -public class CitusTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public CitusTLPWhereOracle(CitusGlobalState state) { - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) - .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) - .with(CitusCommon.getCitusErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/clickhouse/ClickHouseOptions.java b/src/sqlancer/clickhouse/ClickHouseOptions.java index 94297ad20..fec8b62a7 100644 --- a/src/sqlancer/clickhouse/ClickHouseOptions.java +++ b/src/sqlancer/clickhouse/ClickHouseOptions.java @@ -1,6 +1,5 @@ package sqlancer.clickhouse; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,16 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.clickhouse.ClickHouseOptions.ClickHouseOracleFactory; -import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; -import sqlancer.clickhouse.oracle.norec.ClickHouseNoRECOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPHavingOracle; -import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPWhereOracle; -import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "ClickHouse (default port: " + ClickHouseOptions.DEFAULT_PORT + ", default host: " + ClickHouseOptions.DEFAULT_HOST + ")") @@ -34,46 +23,6 @@ public class ClickHouseOptions implements DBMSSpecificOptions { - TLPWhere { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPWhereOracle(globalState); - } - }, - TLPDistinct { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPDistinctOracle(globalState); - } - }, - TLPGroupBy { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPGroupByOracle(globalState); - } - }, - TLPAggregate { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPAggregateOracle(globalState); - } - }, - TLPHaving { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseTLPHavingOracle(globalState); - } - }, - NoREC { - @Override - public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseNoRECOracle(globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return oracle; diff --git a/src/sqlancer/clickhouse/ClickHouseOracleFactory.java b/src/sqlancer/clickhouse/ClickHouseOracleFactory.java new file mode 100644 index 000000000..53b38a6f1 --- /dev/null +++ b/src/sqlancer/clickhouse/ClickHouseOracleFactory.java @@ -0,0 +1,62 @@ +package sqlancer.clickhouse; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; +import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPAggregateOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPDistinctOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPGroupByOracle; +import sqlancer.clickhouse.oracle.tlp.ClickHouseTLPHavingOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; + +public enum ClickHouseOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(ClickHouseErrors.getExpectedExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + TLPDistinct { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPDistinctOracle(globalState); + } + }, + TLPGroupBy { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPGroupByOracle(globalState); + } + }, + TLPAggregate { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPAggregateOracle(globalState); + } + }, + TLPHaving { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + return new ClickHouseTLPHavingOracle(globalState); + } + }, + NoREC { + @Override + public TestOracle create(ClickHouseGlobalState globalState) throws SQLException { + ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + + return new NoRECOracle<>(globalState, gen, errors); + } + } +} diff --git a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java b/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java deleted file mode 100644 index d2acc24d6..000000000 --- a/src/sqlancer/clickhouse/oracle/norec/ClickHouseNoRECOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.clickhouse.oracle.norec; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.clickhouse.ClickHouseErrors; -import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; -import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ast.ClickHouseExpression; -import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; -import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class ClickHouseNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public ClickHouseNoRECOracle(ClickHouseGlobalState globalState) { - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - -} diff --git a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java b/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java deleted file mode 100644 index 564db22f7..000000000 --- a/src/sqlancer/clickhouse/oracle/tlp/ClickHouseTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.clickhouse.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.clickhouse.ClickHouseErrors; -import sqlancer.clickhouse.ClickHouseProvider.ClickHouseGlobalState; -import sqlancer.clickhouse.ClickHouseSchema; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseColumn; -import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; -import sqlancer.clickhouse.ast.ClickHouseExpression; -import sqlancer.clickhouse.ast.ClickHouseExpression.ClickHouseJoin; -import sqlancer.clickhouse.ast.ClickHouseSelect; -import sqlancer.clickhouse.gen.ClickHouseExpressionGenerator; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class ClickHouseTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public ClickHouseTLPWhereOracle(ClickHouseGlobalState state) { - ClickHouseExpressionGenerator gen = new ClickHouseExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(ClickHouseErrors.getExpectedExpressionErrors()) - .build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/cnosdb/CnosDBOptions.java b/src/sqlancer/cnosdb/CnosDBOptions.java index 9a6da00c4..f101c2d38 100644 --- a/src/sqlancer/cnosdb/CnosDBOptions.java +++ b/src/sqlancer/cnosdb/CnosDBOptions.java @@ -1,20 +1,11 @@ package sqlancer.cnosdb; -import java.util.ArrayList; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.cnosdb.CnosDBOptions.CnosDBOracleFactory; -import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPAggregateOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPHavingOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPWhereOracle; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "CnosDB (default port: " + CnosDBOptions.DEFAULT_PORT + ", default host: " + CnosDBOptions.DEFAULT_HOST + ")") @@ -34,31 +25,4 @@ public List getTestOracleFactory() { return oracle; } - public enum CnosDBOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBNoRECOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - List> oracles = new ArrayList<>(); - oracles.add(new CnosDBTLPWhereOracle(globalState)); - oracles.add(new CnosDBTLPHavingOracle(globalState)); - oracles.add(new CnosDBTLPAggregateOracle(globalState)); - return new CompositeTestOracle<>(oracles, globalState); - } - } - - } - } diff --git a/src/sqlancer/cnosdb/CnosDBOracleFactory.java b/src/sqlancer/cnosdb/CnosDBOracleFactory.java new file mode 100644 index 000000000..7cb9c4fc6 --- /dev/null +++ b/src/sqlancer/cnosdb/CnosDBOracleFactory.java @@ -0,0 +1,39 @@ +package sqlancer.cnosdb; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPAggregateOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPHavingOracle; +import sqlancer.cnosdb.oracle.tlp.CnosDBTLPWhereOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TestOracle; + +public enum CnosDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + return new CnosDBNoRECOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + return new CnosDBTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(CnosDBGlobalState globalState) { + List> oracles = new ArrayList<>(); + oracles.add(new CnosDBTLPWhereOracle(globalState)); + oracles.add(new CnosDBTLPHavingOracle(globalState)); + oracles.add(new CnosDBTLPAggregateOracle(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index e50a30c51..ce8a207d6 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -1,7 +1,5 @@ package sqlancer.cockroachdb; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,19 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; -import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPWhereOracle; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; @Parameters(separators = "=", commandDescription = "CockroachDB (default port: " + CockroachDBOptions.DEFAULT_PORT + " default host: " + CockroachDBOptions.DEFAULT_HOST + ")") @@ -32,78 +17,6 @@ public class CockroachDBOptions implements DBMSSpecificOptions { - NOREC { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBNoRECOracle(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPAggregateOracle(globalState); - } - - }, - GROUP_BY { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPGroupByOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPHavingOracle(globalState); - } - }, - WHERE { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPWhereOracle(globalState); - } - }, - DISTINCT { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPDistinctOracle(globalState); - } - }, - EXTENDED_WHERE { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBTLPExtendedWhereOracle(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new CockroachDBTLPAggregateOracle(globalState)); - oracles.add(new CockroachDBTLPHavingOracle(globalState)); - oracles.add(new CockroachDBTLPWhereOracle(globalState)); - oracles.add(new CockroachDBTLPGroupByOracle(globalState)); - oracles.add(new CockroachDBTLPExtendedWhereOracle(globalState)); - oracles.add(new CockroachDBTLPDistinctOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - CERT { - @Override - public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBCERTOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }; - - } - @Parameter(names = { "--test-hash-indexes" }, description = "Test the USING HASH WITH BUCKET_COUNT=n_buckets option in CREATE INDEX") public boolean testHashIndexes = true; diff --git a/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java new file mode 100644 index 000000000..bbf261f16 --- /dev/null +++ b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java @@ -0,0 +1,109 @@ +package sqlancer.cockroachdb; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; + +public enum CockroachDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with(CockroachDBErrors.getTransactionErrors()).with("unable to vectorize execution plan") // SET + // vectorize=experimental_always; + .with(" mismatched physical types at index") // SET vectorize=experimental_always; + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + AGGREGATE { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPAggregateOracle(globalState); + } + + }, + GROUP_BY { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPGroupByOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPHavingOracle(globalState); + } + }, + WHERE { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .with("GROUP BY term out of range").build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + DISTINCT { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPDistinctOracle(globalState); + } + }, + EXTENDED_WHERE { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBTLPExtendedWhereOracle(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(WHERE.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + oracles.add(EXTENDED_WHERE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CERT { + @Override + public TestOracle create( + CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }; + +} diff --git a/src/sqlancer/cockroachdb/CockroachDBProvider.java b/src/sqlancer/cockroachdb/CockroachDBProvider.java index 494c63a5b..66e80fd59 100644 --- a/src/sqlancer/cockroachdb/CockroachDBProvider.java +++ b/src/sqlancer/cockroachdb/CockroachDBProvider.java @@ -19,7 +19,6 @@ import sqlancer.SQLConnection; import sqlancer.SQLGlobalState; import sqlancer.SQLProviderAdapter; -import sqlancer.cockroachdb.CockroachDBOptions.CockroachDBOracleFactory; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.gen.CockroachDBCommentOnGenerator; diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java index 09f5b165d..880f33310 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBRandomQuerySynthesizer.java @@ -13,7 +13,7 @@ import sqlancer.cockroachdb.ast.CockroachDBExpression; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; +import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPBase; import sqlancer.common.query.SQLQueryAdapter; public final class CockroachDBRandomQuerySynthesizer { @@ -49,7 +49,7 @@ public static CockroachDBSelect generateSelect(CockroachDBGlobalState globalStat .map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList()); List updatedTableList = CockroachDBCommon.getTableReferences(tableList); if (Randomly.getBoolean()) { - select.setJoinList(CockroachDBNoRECOracle.getJoins(updatedTableList, globalState)); + select.setJoinList(CockroachDBTLPBase.getJoins(updatedTableList, globalState)); } select.setFromList(updatedTableList); if (Randomly.getBoolean()) { diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java deleted file mode 100644 index 3d0422dfd..000000000 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBNoRECOracle.java +++ /dev/null @@ -1,69 +0,0 @@ -package sqlancer.cockroachdb.oracle; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.Reproducer; -import sqlancer.cockroachdb.CockroachDBErrors; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBSelect; -import sqlancer.cockroachdb.ast.CockroachDBTableReference; -import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class CockroachDBNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public CockroachDBNoRECOracle(CockroachDBGlobalState globalState) { - CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) - .with(CockroachDBErrors.getTransactionErrors()).with("unable to vectorize execution plan") // SET - // vectorize=experimental_always; - .with(" mismatched physical types at index") // SET vectorize=experimental_always; - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - public static List getJoins(List tableList, - CockroachDBGlobalState globalState) throws AssertionError { - List joinExpressions = new ArrayList<>(); - while (tableList.size() >= 2 && Randomly.getBoolean()) { - CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); - CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); - List columns = new ArrayList<>(leftTable.getTable().getColumns()); - columns.addAll(rightTable.getTable().getColumns()); - CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) - .setColumns(columns); - joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), - joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); - } - return joinExpressions; - } - -} diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java index c9034eb65..6dd4bd5d5 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPAggregateOracle.java @@ -29,7 +29,6 @@ import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; @@ -72,7 +71,7 @@ public void check() throws SQLException { .map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList()); List from = CockroachDBCommon.getTableReferences(tableList); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setJoinList(CockroachDBNoRECOracle.getJoins(from, state)); + select.setJoinList(CockroachDBTLPBase.getJoins(from, state)); } select.setFromList(from); if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index 21914eae2..c4809d231 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -10,14 +10,15 @@ import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; import sqlancer.cockroachdb.ast.CockroachDBColumnReference; import sqlancer.cockroachdb.ast.CockroachDBExpression; +import sqlancer.cockroachdb.ast.CockroachDBJoin; import sqlancer.cockroachdb.ast.CockroachDBSelect; import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBNoRECOracle; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; import sqlancer.common.oracle.TestOracle; @@ -47,7 +48,7 @@ public void check() throws SQLException { List tables = targetTables.getTables(); List tableList = tables.stream().map(t -> new CockroachDBTableReference(t)) .collect(Collectors.toList()); - List joins = CockroachDBNoRECOracle.getJoins(tableList, state); + List joins = getJoins(tableList, state); select.setJoinList(joins); select.setFromList(tableList); select.setWhereClause(null); @@ -69,4 +70,20 @@ protected ExpressionGenerator getGen() { return gen; } + public static List getJoins(List tableList, + CockroachDBGlobalState globalState) throws AssertionError { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBoolean()) { + CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); + CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftTable, rightTable, CockroachDBJoin.JoinType.getRandom(), + joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + return joinExpressions; + } + } diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java deleted file mode 100644 index 5fc14c492..000000000 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.cockroachdb.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.cockroachdb.CockroachDBErrors; -import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; -import sqlancer.cockroachdb.CockroachDBSchema; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; -import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; -import sqlancer.cockroachdb.ast.CockroachDBExpression; -import sqlancer.cockroachdb.ast.CockroachDBJoin; -import sqlancer.cockroachdb.ast.CockroachDBSelect; -import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; - -public class CockroachDBTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public CockroachDBTLPWhereOracle(CockroachDBGlobalState state) { - CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) - .with("GROUP BY term out of range").build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/databend/DatabendOptions.java b/src/sqlancer/databend/DatabendOptions.java index ed597066b..d38ee0d59 100644 --- a/src/sqlancer/databend/DatabendOptions.java +++ b/src/sqlancer/databend/DatabendOptions.java @@ -1,7 +1,5 @@ package sqlancer.databend; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,18 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.databend.DatabendOptions.DatabendOracleFactory; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.test.DatabendNoRECOracle; -import sqlancer.databend.test.DatabendPivotedQuerySynthesisOracle; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningAggregateTester; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningDistinctTester; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningGroupByTester; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningHavingTester; -import sqlancer.databend.test.tlp.DatabendQueryPartitioningWhereTester; @Parameters(commandDescription = "Databend") public class DatabendOptions implements DBMSSpecificOptions { @@ -96,68 +82,6 @@ public class DatabendOptions implements DBMSSpecificOptions oracles = Arrays.asList(DatabendOracleFactory.QUERY_PARTITIONING); - public enum DatabendOracleFactory implements OracleFactory { - NOREC { - - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendNoRECOracle(globalState); - } - - }, - HAVING { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningAggregateTester(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - return new DatabendQueryPartitioningDistinctTester(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(DatabendGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new DatabendQueryPartitioningWhereTester(globalState)); - oracles.add(new DatabendQueryPartitioningHavingTester(globalState)); - oracles.add(new DatabendQueryPartitioningAggregateTester(globalState)); - oracles.add(new DatabendQueryPartitioningDistinctTester(globalState)); - oracles.add(new DatabendQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - PQS { - @Override - public TestOracle create(DatabendGlobalState globalState) throws Exception { - return new DatabendPivotedQuerySynthesisOracle(globalState); - } - } - - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/databend/DatabendOracleFactory.java b/src/sqlancer/databend/DatabendOracleFactory.java new file mode 100644 index 000000000..ea66bc353 --- /dev/null +++ b/src/sqlancer/databend/DatabendOracleFactory.java @@ -0,0 +1,93 @@ +package sqlancer.databend; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.databend.gen.DatabendNewExpressionGenerator; +import sqlancer.databend.test.DatabendPivotedQuerySynthesisOracle; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningAggregateTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningDistinctTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningGroupByTester; +import sqlancer.databend.test.tlp.DatabendQueryPartitioningHavingTester; + +public enum DatabendOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) + .with(DatabendErrors.getGroupByErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + GROUP_BY { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws SQLException { + return new DatabendQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + PQS { + @Override + public TestOracle create(DatabendProvider.DatabendGlobalState globalState) + throws Exception { + return new DatabendPivotedQuerySynthesisOracle(globalState); + } + } + +} diff --git a/src/sqlancer/databend/test/DatabendNoRECOracle.java b/src/sqlancer/databend/test/DatabendNoRECOracle.java deleted file mode 100644 index 4d0d29711..000000000 --- a/src/sqlancer/databend/test/DatabendNoRECOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.databend.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.ast.DatabendJoin; -import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.gen.DatabendNewExpressionGenerator; - -public class DatabendNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public DatabendNoRECOracle(DatabendGlobalState globalState) { - DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - -} diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java deleted file mode 100644 index b6520e7f2..000000000 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningWhereTester.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.databend.test.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.databend.DatabendErrors; -import sqlancer.databend.DatabendProvider.DatabendGlobalState; -import sqlancer.databend.DatabendSchema; -import sqlancer.databend.DatabendSchema.DatabendColumn; -import sqlancer.databend.DatabendSchema.DatabendTable; -import sqlancer.databend.ast.DatabendExpression; -import sqlancer.databend.ast.DatabendJoin; -import sqlancer.databend.ast.DatabendSelect; -import sqlancer.databend.gen.DatabendNewExpressionGenerator; - -public class DatabendQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public DatabendQueryPartitioningWhereTester(DatabendGlobalState state) { - DatabendNewExpressionGenerator gen = new DatabendNewExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DatabendErrors.getExpressionErrors()) - .with(DatabendErrors.getGroupByErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/datafusion/DataFusionOptions.java b/src/sqlancer/datafusion/DataFusionOptions.java index fcb0221a6..582b2f658 100644 --- a/src/sqlancer/datafusion/DataFusionOptions.java +++ b/src/sqlancer/datafusion/DataFusionOptions.java @@ -1,6 +1,5 @@ package sqlancer.datafusion; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,12 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.datafusion.DataFusionOptions.DataFusionOracleFactory; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.test.DataFusionNoRECOracle; -import sqlancer.datafusion.test.DataFusionQueryPartitioningWhereTester; @Parameters(commandDescription = "DataFusion") public class DataFusionOptions implements DBMSSpecificOptions { @@ -25,19 +18,4 @@ public List getTestOracleFactory() { return Arrays.asList(DataFusionOracleFactory.NOREC, DataFusionOracleFactory.QUERY_PARTITIONING_WHERE); } - public enum DataFusionOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(DataFusionGlobalState globalState) throws SQLException { - return new DataFusionNoRECOracle(globalState); - } - }, - QUERY_PARTITIONING_WHERE { - @Override - public TestOracle create(DataFusionGlobalState globalState) throws SQLException { - return new DataFusionQueryPartitioningWhereTester(globalState); - } - } - } - } diff --git a/src/sqlancer/datafusion/DataFusionOracleFactory.java b/src/sqlancer/datafusion/DataFusionOracleFactory.java new file mode 100644 index 000000000..a7a3f21e8 --- /dev/null +++ b/src/sqlancer/datafusion/DataFusionOracleFactory.java @@ -0,0 +1,34 @@ +package sqlancer.datafusion; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.datafusion.gen.DataFusionExpressionGenerator; + +public enum DataFusionOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create( + DataFusionProvider.DataFusionGlobalState globalState) throws SQLException { + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + QUERY_PARTITIONING_WHERE { + @Override + public TestOracle create( + DataFusionProvider.DataFusionGlobalState globalState) throws SQLException { + DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(DataFusionErrors.getExpectedExecutionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + } +} diff --git a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java b/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java deleted file mode 100644 index 36ab97854..000000000 --- a/src/sqlancer/datafusion/test/DataFusionNoRECOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.datafusion.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.datafusion.DataFusionErrors; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionSchema; -import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; -import sqlancer.datafusion.DataFusionSchema.DataFusionTable; -import sqlancer.datafusion.ast.DataFusionExpression; -import sqlancer.datafusion.ast.DataFusionJoin; -import sqlancer.datafusion.ast.DataFusionSelect; -import sqlancer.datafusion.gen.DataFusionExpressionGenerator; - -public class DataFusionNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public DataFusionNoRECOracle(DataFusionGlobalState globalState) { - DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java b/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java deleted file mode 100644 index e0335a18f..000000000 --- a/src/sqlancer/datafusion/test/DataFusionQueryPartitioningWhereTester.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.datafusion.test; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.datafusion.DataFusionErrors; -import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; -import sqlancer.datafusion.DataFusionSchema; -import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; -import sqlancer.datafusion.DataFusionSchema.DataFusionTable; -import sqlancer.datafusion.ast.DataFusionExpression; -import sqlancer.datafusion.ast.DataFusionJoin; -import sqlancer.datafusion.ast.DataFusionSelect; -import sqlancer.datafusion.gen.DataFusionExpressionGenerator; - -public class DataFusionQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public DataFusionQueryPartitioningWhereTester(DataFusionGlobalState state) { - DataFusionExpressionGenerator gen = new DataFusionExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DataFusionErrors.getExpectedExecutionErrors()) - .build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/doris/DorisOptions.java b/src/sqlancer/doris/DorisOptions.java index a2a05b95e..7e5a2c7e3 100644 --- a/src/sqlancer/doris/DorisOptions.java +++ b/src/sqlancer/doris/DorisOptions.java @@ -1,7 +1,5 @@ package sqlancer.doris; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,18 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.doris.DorisOptions.DorisOracleFactory; -import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.oracle.DorisNoRECOracle; -import sqlancer.doris.oracle.DorisPivotedQuerySynthesisOracle; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningAggregateTester; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningDistinctTester; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningGroupByTester; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningHavingTester; -import sqlancer.doris.oracle.tlp.DorisQueryPartitioningWhereTester; @Parameters(commandDescription = "Apache Doris (default port: " + DorisOptions.DEFAULT_PORT + ", default host: " + DorisOptions.DEFAULT_HOST + ")") @@ -112,80 +98,6 @@ public class DorisOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracles = Arrays.asList(DorisOracleFactory.NOREC); - public enum DorisOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisNoRECOracle(globalState); - } - - }, - HAVING { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningAggregateTester(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - return new DorisQueryPartitioningDistinctTester(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(DorisGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new DorisQueryPartitioningWhereTester(globalState)); - oracles.add(new DorisQueryPartitioningHavingTester(globalState)); - oracles.add(new DorisQueryPartitioningAggregateTester(globalState)); - oracles.add(new DorisQueryPartitioningDistinctTester(globalState)); - oracles.add(new DorisQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - PQS { - @Override - public TestOracle create(DorisGlobalState globalState) throws Exception { - return new DorisPivotedQuerySynthesisOracle(globalState); - } - }, - ALL { - @Override - public TestOracle create(DorisGlobalState globalState) throws Exception { - List> oracles = new ArrayList<>(); - oracles.add(new DorisNoRECOracle(globalState)); - oracles.add(new DorisQueryPartitioningWhereTester(globalState)); - oracles.add(new DorisQueryPartitioningHavingTester(globalState)); - oracles.add(new DorisQueryPartitioningAggregateTester(globalState)); - oracles.add(new DorisQueryPartitioningDistinctTester(globalState)); - oracles.add(new DorisQueryPartitioningGroupByTester(globalState)); - oracles.add(new DorisPivotedQuerySynthesisOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - } - - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/doris/DorisOracleFactory.java b/src/sqlancer/doris/DorisOracleFactory.java new file mode 100644 index 000000000..8f28f0f21 --- /dev/null +++ b/src/sqlancer/doris/DorisOracleFactory.java @@ -0,0 +1,108 @@ +package sqlancer.doris; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.doris.gen.DorisNewExpressionGenerator; +import sqlancer.doris.oracle.DorisPivotedQuerySynthesisOracle; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningAggregateTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningDistinctTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningGroupByTester; +import sqlancer.doris.oracle.tlp.DorisQueryPartitioningHavingTester; + +public enum DorisOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) + .with(DorisErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + GROUP_BY { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws SQLException { + return new DorisQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + PQS { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws Exception { + return new DorisPivotedQuerySynthesisOracle(globalState); + } + }, + ALL { + @Override + public TestOracle create(DorisProvider.DorisGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(NOREC.create(globalState)); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + oracles.add(new DorisPivotedQuerySynthesisOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/doris/oracle/DorisNoRECOracle.java b/src/sqlancer/doris/oracle/DorisNoRECOracle.java deleted file mode 100644 index 9ce94b952..000000000 --- a/src/sqlancer/doris/oracle/DorisNoRECOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.doris.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.doris.DorisErrors; -import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.DorisSchema; -import sqlancer.doris.DorisSchema.DorisColumn; -import sqlancer.doris.DorisSchema.DorisTable; -import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.ast.DorisJoin; -import sqlancer.doris.ast.DorisSelect; -import sqlancer.doris.gen.DorisNewExpressionGenerator; - -public class DorisNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public DorisNoRECOracle(DorisGlobalState globalState) { - DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java b/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java deleted file mode 100644 index 32b171af0..000000000 --- a/src/sqlancer/doris/oracle/tlp/DorisQueryPartitioningWhereTester.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.doris.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.doris.DorisErrors; -import sqlancer.doris.DorisProvider.DorisGlobalState; -import sqlancer.doris.DorisSchema; -import sqlancer.doris.DorisSchema.DorisColumn; -import sqlancer.doris.DorisSchema.DorisTable; -import sqlancer.doris.ast.DorisExpression; -import sqlancer.doris.ast.DorisJoin; -import sqlancer.doris.ast.DorisSelect; -import sqlancer.doris.gen.DorisNewExpressionGenerator; - -public class DorisQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public DorisQueryPartitioningWhereTester(DorisGlobalState state) { - DorisNewExpressionGenerator gen = new DorisNewExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DorisErrors.getExpressionErrors()) - .with(DorisErrors.getExpressionErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index f14379163..00f85eece 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -1,7 +1,5 @@ package sqlancer.duckdb; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,17 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.duckdb.DuckDBOptions.DuckDBOracleFactory; -import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.test.DuckDBNoRECOracle; -import sqlancer.duckdb.test.DuckDBQueryPartitioningAggregateTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningDistinctTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningGroupByTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningHavingTester; -import sqlancer.duckdb.test.DuckDBQueryPartitioningWhereTester; @Parameters(commandDescription = "DuckDB") public class DuckDBOptions implements DBMSSpecificOptions { @@ -93,62 +80,6 @@ public class DuckDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracles = Arrays.asList(DuckDBOracleFactory.QUERY_PARTITIONING); - public enum DuckDBOracleFactory implements OracleFactory { - NOREC { - - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBNoRECOracle(globalState); - } - - }, - HAVING { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningAggregateTester(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - return new DuckDBQueryPartitioningDistinctTester(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(DuckDBGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new DuckDBQueryPartitioningWhereTester(globalState)); - oracles.add(new DuckDBQueryPartitioningHavingTester(globalState)); - oracles.add(new DuckDBQueryPartitioningAggregateTester(globalState)); - oracles.add(new DuckDBQueryPartitioningDistinctTester(globalState)); - oracles.add(new DuckDBQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/duckdb/DuckDBOracleFactory.java b/src/sqlancer/duckdb/DuckDBOracleFactory.java new file mode 100644 index 000000000..8fd0f96af --- /dev/null +++ b/src/sqlancer/duckdb/DuckDBOracleFactory.java @@ -0,0 +1,86 @@ +package sqlancer.duckdb; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator; +import sqlancer.duckdb.test.DuckDBQueryPartitioningAggregateTester; +import sqlancer.duckdb.test.DuckDBQueryPartitioningDistinctTester; +import sqlancer.duckdb.test.DuckDBQueryPartitioningGroupByTester; +import sqlancer.duckdb.test.DuckDBQueryPartitioningHavingTester; + +public enum DuckDBOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .withRegex(DuckDBErrors.getExpressionErrorsRegex()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) + .with(DuckDBErrors.getGroupByErrors()).withRegex(DuckDBErrors.getExpressionErrorsRegex()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + GROUP_BY { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws SQLException { + return new DuckDBQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(DuckDBProvider.DuckDBGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java b/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java deleted file mode 100644 index 4096714ac..000000000 --- a/src/sqlancer/duckdb/test/DuckDBNoRECOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.duckdb.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.duckdb.DuckDBErrors; -import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBSchema; -import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; -import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.ast.DuckDBExpression; -import sqlancer.duckdb.ast.DuckDBJoin; -import sqlancer.duckdb.ast.DuckDBSelect; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator; - -public class DuckDBNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public DuckDBNoRECOracle(DuckDBGlobalState globalState) { - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) - .withRegex(DuckDBErrors.getExpressionErrorsRegex()).with("canceling statement due to statement timeout") - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java b/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java deleted file mode 100644 index d4ffa74c8..000000000 --- a/src/sqlancer/duckdb/test/DuckDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.duckdb.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.duckdb.DuckDBErrors; -import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; -import sqlancer.duckdb.DuckDBSchema; -import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; -import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -import sqlancer.duckdb.ast.DuckDBExpression; -import sqlancer.duckdb.ast.DuckDBJoin; -import sqlancer.duckdb.ast.DuckDBSelect; -import sqlancer.duckdb.gen.DuckDBExpressionGenerator; - -public class DuckDBQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public DuckDBQueryPartitioningWhereTester(DuckDBGlobalState state) { - DuckDBExpressionGenerator gen = new DuckDBExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(DuckDBErrors.getExpressionErrors()) - .with(DuckDBErrors.getGroupByErrors()).withRegex(DuckDBErrors.getExpressionErrorsRegex()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/h2/H2Options.java b/src/sqlancer/h2/H2Options.java index e7141fcf2..985853e94 100644 --- a/src/sqlancer/h2/H2Options.java +++ b/src/sqlancer/h2/H2Options.java @@ -1,33 +1,15 @@ package sqlancer.h2; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.h2.H2Options.H2OracleFactory; -import sqlancer.h2.H2Provider.H2GlobalState; @Parameters(commandDescription = "H2") public class H2Options implements DBMSSpecificOptions { - public enum H2OracleFactory implements OracleFactory { - - TLP_WHERE { - - @Override - public TestOracle create(H2GlobalState globalState) throws SQLException { - return new H2QueryPartitioningWhereTester(globalState); - } - - }; - - } - @Override public List getTestOracleFactory() { return Arrays.asList(H2OracleFactory.TLP_WHERE); diff --git a/src/sqlancer/h2/H2OracleFactory.java b/src/sqlancer/h2/H2OracleFactory.java new file mode 100644 index 000000000..fbbaade05 --- /dev/null +++ b/src/sqlancer/h2/H2OracleFactory.java @@ -0,0 +1,23 @@ +package sqlancer.h2; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; + +public enum H2OracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(H2Provider.H2GlobalState globalState) throws SQLException { + H2ExpressionGenerator gen = new H2ExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(H2Errors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }; + +} diff --git a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java b/src/sqlancer/h2/H2QueryPartitioningWhereTester.java deleted file mode 100644 index f1bb73dfa..000000000 --- a/src/sqlancer/h2/H2QueryPartitioningWhereTester.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.h2; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.h2.H2Provider.H2GlobalState; -import sqlancer.h2.H2Schema.H2Column; -import sqlancer.h2.H2Schema.H2Table; -import sqlancer.h2.ast.H2Expression; -import sqlancer.h2.ast.H2Join; -import sqlancer.h2.ast.H2Select; - -public class H2QueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public H2QueryPartitioningWhereTester(H2GlobalState state) { - H2ExpressionGenerator gen = new H2ExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(H2Errors.getExpressionErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/hsqldb/HSQLDBOptions.java b/src/sqlancer/hsqldb/HSQLDBOptions.java index 3e3dcc4dc..215f4f0fc 100644 --- a/src/sqlancer/hsqldb/HSQLDBOptions.java +++ b/src/sqlancer/hsqldb/HSQLDBOptions.java @@ -1,40 +1,17 @@ package sqlancer.hsqldb; -import java.sql.SQLException; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.hsqldb.test.HSQLDBNoRECOracle; -import sqlancer.hsqldb.test.HSQLDBQueryPartitioningWhereTester; @Parameters(commandDescription = "hsqldb") -public class HSQLDBOptions implements DBMSSpecificOptions { +public class HSQLDBOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") - public List oracle = List.of(HSQLDBOracleFactory.WHERE, - HSQLDBOracleFactory.NOREC); - - public enum HSQLDBOracleFactory implements OracleFactory { - WHERE { - @Override - public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) - throws SQLException { - return new HSQLDBQueryPartitioningWhereTester(globalState); - } - }, - NOREC { - @Override - public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) - throws Exception { - return new HSQLDBNoRECOracle(globalState); - } - } - } + public List oracle = List.of(HSQLDBOracleFactory.WHERE, HSQLDBOracleFactory.NOREC); @Override public List getTestOracleFactory() { diff --git a/src/sqlancer/hsqldb/HSQLDBOracleFactory.java b/src/sqlancer/hsqldb/HSQLDBOracleFactory.java new file mode 100644 index 000000000..ed4fdf79c --- /dev/null +++ b/src/sqlancer/hsqldb/HSQLDBOracleFactory.java @@ -0,0 +1,32 @@ +package sqlancer.hsqldb; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; + +public enum HSQLDBOracleFactory implements OracleFactory { + WHERE { + @Override + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws SQLException { + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + NOREC { + @Override + public TestOracle create(HSQLDBProvider.HSQLDBGlobalState globalState) + throws Exception { + HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); + return new NoRECOracle<>(globalState, gen, errors); + } + } +} diff --git a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java b/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java deleted file mode 100644 index 0b1a3b8b8..000000000 --- a/src/sqlancer/hsqldb/test/HSQLDBNoRECOracle.java +++ /dev/null @@ -1,43 +0,0 @@ -package sqlancer.hsqldb.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.hsqldb.HSQLDBErrors; -import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; -import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; -import sqlancer.hsqldb.ast.HSQLDBExpression; -import sqlancer.hsqldb.ast.HSQLDBJoin; -import sqlancer.hsqldb.ast.HSQLDBSelect; -import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; - -public class HSQLDBNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public HSQLDBNoRECOracle(HSQLDBGlobalState globalState) { - HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java b/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java deleted file mode 100644 index 795c6656c..000000000 --- a/src/sqlancer/hsqldb/test/HSQLDBQueryPartitioningWhereTester.java +++ /dev/null @@ -1,38 +0,0 @@ -package sqlancer.hsqldb.test; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.hsqldb.HSQLDBErrors; -import sqlancer.hsqldb.HSQLDBProvider.HSQLDBGlobalState; -import sqlancer.hsqldb.HSQLDBSchema; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBColumn; -import sqlancer.hsqldb.HSQLDBSchema.HSQLDBTable; -import sqlancer.hsqldb.ast.HSQLDBExpression; -import sqlancer.hsqldb.ast.HSQLDBJoin; -import sqlancer.hsqldb.ast.HSQLDBSelect; -import sqlancer.hsqldb.gen.HSQLDBExpressionGenerator; - -public class HSQLDBQueryPartitioningWhereTester implements TestOracle { - - private final TLPWhereOracle oracle; - - public HSQLDBQueryPartitioningWhereTester(HSQLDBGlobalState state) { - HSQLDBExpressionGenerator gen = new HSQLDBExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HSQLDBErrors.getExpressionErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/mariadb/MariaDBOptions.java b/src/sqlancer/mariadb/MariaDBOptions.java index f601ab0a6..b1e7f807e 100644 --- a/src/sqlancer/mariadb/MariaDBOptions.java +++ b/src/sqlancer/mariadb/MariaDBOptions.java @@ -1,6 +1,5 @@ package sqlancer.mariadb; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,12 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mariadb.MariaDBOptions.MariaDBOracleFactory; -import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; -import sqlancer.mariadb.oracle.MariaDBDQPOracle; -import sqlancer.mariadb.oracle.MariaDBNoRECOracle; @Parameters(separators = "=", commandDescription = "MariaDB (default port: " + MariaDBOptions.DEFAULT_PORT + ", default host: " + MariaDBOptions.DEFAULT_HOST + ")") @@ -24,24 +17,6 @@ public class MariaDBOptions implements DBMSSpecificOptions @Parameter(names = "--oracle") public List oracles = Arrays.asList(MariaDBOracleFactory.NOREC); - public enum MariaDBOracleFactory implements OracleFactory { - - NOREC { - - @Override - public TestOracle create(MariaDBGlobalState globalState) throws SQLException { - return new MariaDBNoRECOracle(globalState); - } - - }, - DQP { - @Override - public TestOracle create(MariaDBGlobalState globalState) throws SQLException { - return new MariaDBDQPOracle(globalState); - } - } - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/mariadb/MariaDBOracleFactory.java b/src/sqlancer/mariadb/MariaDBOracleFactory.java new file mode 100644 index 000000000..549da27c8 --- /dev/null +++ b/src/sqlancer/mariadb/MariaDBOracleFactory.java @@ -0,0 +1,36 @@ +package sqlancer.mariadb; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.mariadb.gen.MariaDBExpressionGenerator; +import sqlancer.mariadb.oracle.MariaDBDQPOracle; + +public enum MariaDBOracleFactory implements OracleFactory { + + NOREC { + @Override + public TestOracle create(MariaDBProvider.MariaDBGlobalState globalState) + throws SQLException { + MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(globalState.getRandomly()); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MariaDBErrors.getCommonErrors()) + .with("is out of range").with("unmatched parentheses").with("nothing to repeat at offset") + .with("missing )").with("missing terminating ]").with("range out of order in character class") + .with("unrecognized character after ").with("Got error '(*VERB) not recognized or malformed") + .with("must be followed by").with("malformed number or name after").with("digit expected after") + .with("Could not create a join buffer").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + DQP { + @Override + public TestOracle create(MariaDBProvider.MariaDBGlobalState globalState) + throws SQLException { + return new MariaDBDQPOracle(globalState); + } + } +} diff --git a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java b/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java deleted file mode 100644 index 039abd85f..000000000 --- a/src/sqlancer/mariadb/oracle/MariaDBNoRECOracle.java +++ /dev/null @@ -1,48 +0,0 @@ -package sqlancer.mariadb.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mariadb.MariaDBErrors; -import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; -import sqlancer.mariadb.MariaDBSchema; -import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; -import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -import sqlancer.mariadb.ast.MariaDBExpression; -import sqlancer.mariadb.ast.MariaDBJoin; -import sqlancer.mariadb.ast.MariaDBSelectStatement; -import sqlancer.mariadb.gen.MariaDBExpressionGenerator; - -public class MariaDBNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public MariaDBNoRECOracle(MariaDBGlobalState globalState) { - MariaDBExpressionGenerator gen = new MariaDBExpressionGenerator(globalState.getRandomly()); - ExpectedErrors errors = ExpectedErrors.newErrors().with(MariaDBErrors.getCommonErrors()).with("is out of range") - .with("unmatched parentheses").with("nothing to repeat at offset").with("missing )") - .with("missing terminating ]").with("range out of order in character class") - .with("unrecognized character after ").with("Got error '(*VERB) not recognized or malformed") - .with("must be followed by").with("malformed number or name after").with("digit expected after") - .with("Could not create a join buffer").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/materialize/MaterializeOptions.java b/src/sqlancer/materialize/MaterializeOptions.java index 23191e3a9..104dc9b51 100644 --- a/src/sqlancer/materialize/MaterializeOptions.java +++ b/src/sqlancer/materialize/MaterializeOptions.java @@ -1,7 +1,5 @@ package sqlancer.materialize; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,15 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.materialize.MaterializeOptions.MaterializeOracleFactory; -import sqlancer.materialize.oracle.MaterializeNoRECOracle; -import sqlancer.materialize.oracle.MaterializePivotedQuerySynthesisOracle; -import sqlancer.materialize.oracle.tlp.MaterializeTLPAggregateOracle; -import sqlancer.materialize.oracle.tlp.MaterializeTLPHavingOracle; -import sqlancer.materialize.oracle.tlp.MaterializeTLPWhereOracle; @Parameters(separators = "=", commandDescription = "Materialize (default port: " + MaterializeOptions.DEFAULT_PORT + ", default host: " + MaterializeOptions.DEFAULT_HOST + ", default user: " + MaterializeOptions.DEFAULT_USER @@ -46,45 +35,6 @@ public class MaterializeOptions implements DBMSSpecificOptions { - NOREC { - @Override - public TestOracle create(MaterializeGlobalState globalState) throws SQLException { - return new MaterializeNoRECOracle(globalState); - } - }, - PQS { - @Override - public TestOracle create(MaterializeGlobalState globalState) throws SQLException { - return new MaterializePivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - HAVING { - - @Override - public TestOracle create(MaterializeGlobalState globalState) throws SQLException { - return new MaterializeTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(MaterializeGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new MaterializeTLPWhereOracle(globalState)); - oracles.add(new MaterializeTLPHavingOracle(globalState)); - oracles.add(new MaterializeTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return oracle; diff --git a/src/sqlancer/materialize/MaterializeOracleFactory.java b/src/sqlancer/materialize/MaterializeOracleFactory.java new file mode 100644 index 000000000..c652fd56a --- /dev/null +++ b/src/sqlancer/materialize/MaterializeOracleFactory.java @@ -0,0 +1,70 @@ +package sqlancer.materialize; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.materialize.gen.MaterializeCommon; +import sqlancer.materialize.gen.MaterializeExpressionGenerator; +import sqlancer.materialize.oracle.MaterializePivotedQuerySynthesisOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPAggregateOracle; +import sqlancer.materialize.oracle.tlp.MaterializeTLPHavingOracle; + +public enum MaterializeOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) + .with(MaterializeCommon.getCommonFetchErrors()).with("canceling statement due to statement timeout") + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + WHERE { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(MaterializeCommon.getCommonExpressionErrors()).with(MaterializeCommon.getCommonFetchErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + HAVING { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws SQLException { + return new MaterializeTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(MaterializeGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(new MaterializeTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java index f35e476f3..5b18b1dfb 100644 --- a/src/sqlancer/materialize/MaterializeProvider.java +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -25,7 +25,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.materialize.MaterializeOptions.MaterializeOracleFactory; import sqlancer.materialize.gen.MaterializeDeleteGenerator; import sqlancer.materialize.gen.MaterializeDropIndexGenerator; import sqlancer.materialize.gen.MaterializeIndexGenerator; diff --git a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java b/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java deleted file mode 100644 index e0f665d93..000000000 --- a/src/sqlancer/materialize/oracle/MaterializeNoRECOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.materialize.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.materialize.MaterializeGlobalState; -import sqlancer.materialize.MaterializeSchema; -import sqlancer.materialize.MaterializeSchema.MaterializeColumn; -import sqlancer.materialize.MaterializeSchema.MaterializeTable; -import sqlancer.materialize.ast.MaterializeExpression; -import sqlancer.materialize.ast.MaterializeJoin; -import sqlancer.materialize.ast.MaterializeSelect; -import sqlancer.materialize.gen.MaterializeCommon; -import sqlancer.materialize.gen.MaterializeExpressionGenerator; - -public class MaterializeNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public MaterializeNoRECOracle(MaterializeGlobalState globalState) { - MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) - .with(MaterializeCommon.getCommonFetchErrors()).with("canceling statement due to statement timeout") - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java b/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java deleted file mode 100644 index cd14bfb75..000000000 --- a/src/sqlancer/materialize/oracle/tlp/MaterializeTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.materialize.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.materialize.MaterializeGlobalState; -import sqlancer.materialize.MaterializeSchema; -import sqlancer.materialize.MaterializeSchema.MaterializeColumn; -import sqlancer.materialize.MaterializeSchema.MaterializeTable; -import sqlancer.materialize.ast.MaterializeExpression; -import sqlancer.materialize.ast.MaterializeJoin; -import sqlancer.materialize.ast.MaterializeSelect; -import sqlancer.materialize.gen.MaterializeCommon; -import sqlancer.materialize.gen.MaterializeExpressionGenerator; - -public class MaterializeTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public MaterializeTLPWhereOracle(MaterializeGlobalState state) { - MaterializeExpressionGenerator gen = new MaterializeExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MaterializeCommon.getCommonExpressionErrors()) - .with(MaterializeCommon.getCommonFetchErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 86cf6dd18..10132b57c 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import sqlancer.SQLGlobalState; -import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; public class MySQLGlobalState extends SQLGlobalState { diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 87408c39f..9219073d5 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -1,6 +1,5 @@ package sqlancer.mysql; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,14 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; -import sqlancer.mysql.oracle.MySQLCERTOracle; -import sqlancer.mysql.oracle.MySQLDQPOracle; -import sqlancer.mysql.oracle.MySQLFuzzer; -import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; -import sqlancer.mysql.oracle.MySQLTLPWhereOracle; @Parameters(separators = "=", commandDescription = "MySQL (default port: " + MySQLOptions.DEFAULT_PORT + ", default host: " + MySQLOptions.DEFAULT_HOST + ")") @@ -26,55 +17,6 @@ public class MySQLOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracles = Arrays.asList(MySQLOracleFactory.TLP_WHERE); - public enum MySQLOracleFactory implements OracleFactory { - - TLP_WHERE { - - @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLTLPWhereOracle(globalState); - } - - }, - PQS { - - @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLPivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - - }, - CERT { - @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLCERTOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - FUZZER { - @Override - public TestOracle create(MySQLGlobalState globalState) throws Exception { - return new MySQLFuzzer(globalState); - } - - }, - DQP { - @Override - public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLDQPOracle(globalState); - } - }; - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java new file mode 100644 index 000000000..206179101 --- /dev/null +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -0,0 +1,64 @@ +package sqlancer.mysql; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.mysql.gen.MySQLExpressionGenerator; +import sqlancer.mysql.oracle.MySQLCERTOracle; +import sqlancer.mysql.oracle.MySQLDQPOracle; +import sqlancer.mysql.oracle.MySQLFuzzer; +import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; + +public enum MySQLOracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }, + PQS { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + + }, + CERT { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + FUZZER { + @Override + public TestOracle create(MySQLGlobalState globalState) throws Exception { + return new MySQLFuzzer(globalState); + } + + }, + DQP { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLDQPOracle(globalState); + } + }; +} diff --git a/src/sqlancer/mysql/MySQLProvider.java b/src/sqlancer/mysql/MySQLProvider.java index 287f20a7e..80a23b947 100644 --- a/src/sqlancer/mysql/MySQLProvider.java +++ b/src/sqlancer/mysql/MySQLProvider.java @@ -21,7 +21,6 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; -import sqlancer.mysql.MySQLOptions.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.gen.MySQLAlterTable; diff --git a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java b/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java deleted file mode 100644 index 6d0fd1178..000000000 --- a/src/sqlancer/mysql/oracle/MySQLTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.mysql.MySQLErrors; -import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLSchema; -import sqlancer.mysql.MySQLSchema.MySQLColumn; -import sqlancer.mysql.MySQLSchema.MySQLTable; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLJoin; -import sqlancer.mysql.ast.MySQLSelect; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -public class MySQLTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public MySQLTLPWhereOracle(MySQLGlobalState state) { - MySQLExpressionGenerator gen = new MySQLExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) - .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/oceanbase/OceanBaseGlobalState.java b/src/sqlancer/oceanbase/OceanBaseGlobalState.java index b8baf08fb..fa483072b 100644 --- a/src/sqlancer/oceanbase/OceanBaseGlobalState.java +++ b/src/sqlancer/oceanbase/OceanBaseGlobalState.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import sqlancer.SQLGlobalState; -import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; public class OceanBaseGlobalState extends SQLGlobalState { diff --git a/src/sqlancer/oceanbase/OceanBaseOptions.java b/src/sqlancer/oceanbase/OceanBaseOptions.java index 86507f769..949e5158b 100644 --- a/src/sqlancer/oceanbase/OceanBaseOptions.java +++ b/src/sqlancer/oceanbase/OceanBaseOptions.java @@ -1,6 +1,5 @@ package sqlancer.oceanbase; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,12 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.oceanbase.OceanBaseOptions.OceanBaseOracleFactory; -import sqlancer.oceanbase.oracle.OceanBaseNoRECOracle; -import sqlancer.oceanbase.oracle.OceanBasePivotedQuerySynthesisOracle; -import sqlancer.oceanbase.oracle.OceanBaseTLPWhereOracle; @Parameters(separators = "=", commandDescription = "OceanBase (default port: " + OceanBaseOptions.DEFAULT_PORT + ", default host: " + OceanBaseOptions.DEFAULT_HOST + ")") @@ -24,34 +17,6 @@ public class OceanBaseOptions implements DBMSSpecificOptions oracles = Arrays.asList(OceanBaseOracleFactory.TLP_WHERE); - public enum OceanBaseOracleFactory implements OracleFactory { - - TLP_WHERE { - @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { - return new OceanBaseTLPWhereOracle(globalState); - } - }, - NoREC { - @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { - return new OceanBaseNoRECOracle(globalState); - } - }, - PQS { - - @Override - public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { - return new OceanBasePivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - } - } - @Parameter(names = { "--query-timeout" }, description = "Query timeout") public int queryTimeout = 1000000000; @Parameter(names = { "--transaction-timeout" }, description = "Transaction timeout") diff --git a/src/sqlancer/oceanbase/OceanBaseOracleFactory.java b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java new file mode 100644 index 000000000..b7b115f37 --- /dev/null +++ b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java @@ -0,0 +1,50 @@ +package sqlancer.oceanbase; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; +import sqlancer.oceanbase.oracle.OceanBasePivotedQuerySynthesisOracle; + +public enum OceanBaseOracleFactory implements OracleFactory { + + TLP_WHERE { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).with("value is out of range").build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + NoREC { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()) + .with("canceling statement due to statement timeout").with("unmatched parentheses") + .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") + .with("range out of order in character class").with("unrecognized character after ") + .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") + .with("malformed number or name after").with("digit expected after").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(OceanBaseGlobalState globalState) throws SQLException { + return new OceanBasePivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + } +} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java deleted file mode 100644 index 0ab0e6d2f..000000000 --- a/src/sqlancer/oceanbase/oracle/OceanBaseNoRECOracle.java +++ /dev/null @@ -1,49 +0,0 @@ -package sqlancer.oceanbase.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.oceanbase.OceanBaseErrors; -import sqlancer.oceanbase.OceanBaseGlobalState; -import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.ast.OceanBaseExpression; -import sqlancer.oceanbase.ast.OceanBaseJoin; -import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; - -public class OceanBaseNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public OceanBaseNoRECOracle(OceanBaseGlobalState globalState) { - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) - .withRegex(OceanBaseErrors.getExpressionErrorsRegex()) - .with("canceling statement due to statement timeout").with("unmatched parentheses") - .with("nothing to repeat at offset").with("missing )").with("missing terminating ]") - .with("range out of order in character class").with("unrecognized character after ") - .with("Got error '(*VERB) not recognized or malformed").with("must be followed by") - .with("malformed number or name after").with("digit expected after").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java b/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java deleted file mode 100644 index 08f5fef93..000000000 --- a/src/sqlancer/oceanbase/oracle/OceanBaseTLPWhereOracle.java +++ /dev/null @@ -1,45 +0,0 @@ -package sqlancer.oceanbase.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.oceanbase.OceanBaseErrors; -import sqlancer.oceanbase.OceanBaseGlobalState; -import sqlancer.oceanbase.OceanBaseSchema; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseColumn; -import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; -import sqlancer.oceanbase.ast.OceanBaseExpression; -import sqlancer.oceanbase.ast.OceanBaseJoin; -import sqlancer.oceanbase.ast.OceanBaseSelect; -import sqlancer.oceanbase.gen.OceanBaseExpressionGenerator; - -public class OceanBaseTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public OceanBaseTLPWhereOracle(OceanBaseGlobalState state) { - OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) - .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).with("value is out of range").build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index ced65d5da..659dc4ff6 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -1,7 +1,5 @@ package sqlancer.postgres; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,17 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; -import sqlancer.postgres.oracle.PostgresCERTOracle; -import sqlancer.postgres.oracle.PostgresFuzzer; -import sqlancer.postgres.oracle.PostgresNoRECOracle; -import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; -import sqlancer.postgres.oracle.tlp.PostgresTLPWhereOracle; @Parameters(separators = "=", commandDescription = "PostgreSQL (default port: " + PostgresOptions.DEFAULT_PORT + ", default host: " + PostgresOptions.DEFAULT_HOST + ")") @@ -43,63 +30,6 @@ public class PostgresOptions implements DBMSSpecificOptions { - NOREC { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresNoRECOracle(globalState); - } - }, - PQS { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresPivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - HAVING { - - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new PostgresTLPWhereOracle(globalState)); - oracles.add(new PostgresTLPHavingOracle(globalState)); - oracles.add(new PostgresTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - CERT { - @Override - public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresCERTOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - FUZZER { - @Override - public TestOracle create(PostgresGlobalState globalState) throws Exception { - return new PostgresFuzzer(globalState); - } - - }; - - } - @Override public List getTestOracleFactory() { return oracle; diff --git a/src/sqlancer/postgres/PostgresOracleFactory.java b/src/sqlancer/postgres/PostgresOracleFactory.java new file mode 100644 index 000000000..45e2190fc --- /dev/null +++ b/src/sqlancer/postgres/PostgresOracleFactory.java @@ -0,0 +1,91 @@ +package sqlancer.postgres; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.postgres.gen.PostgresCommon; +import sqlancer.postgres.gen.PostgresExpressionGenerator; +import sqlancer.postgres.oracle.PostgresCERTOracle; +import sqlancer.postgres.oracle.PostgresFuzzer; +import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; +import sqlancer.postgres.oracle.tlp.PostgresTLPHavingOracle; + +public enum PostgresOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()).build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + WHERE { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .with(PostgresCommon.getCommonFetchErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }, + HAVING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(new PostgresTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CERT { + @Override + public TestOracle create(PostgresGlobalState globalState) throws SQLException { + return new PostgresCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + FUZZER { + @Override + public TestOracle create(PostgresGlobalState globalState) throws Exception { + return new PostgresFuzzer(globalState); + } + + }; + +} diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 62c43cd5a..7c86dc628 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -22,7 +22,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.PostgresOptions.PostgresOracleFactory; import sqlancer.postgres.gen.PostgresAlterTableGenerator; import sqlancer.postgres.gen.PostgresAnalyzeGenerator; import sqlancer.postgres.gen.PostgresClusterGenerator; diff --git a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java b/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java deleted file mode 100644 index 9243b4870..000000000 --- a/src/sqlancer/postgres/oracle/PostgresNoRECOracle.java +++ /dev/null @@ -1,79 +0,0 @@ -package sqlancer.postgres.oracle; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresDataType; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.PostgresSchema.PostgresTables; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; -import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; -import sqlancer.postgres.oracle.tlp.PostgresTLPBase; - -public class PostgresNoRECOracle implements TestOracle { - - private final NoRECOracle oracle; - - public PostgresNoRECOracle(PostgresGlobalState globalState) { - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) - .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - public static List getJoinStatements(PostgresGlobalState globalState, List columns, - List tables) { - List joinStatements = new ArrayList<>(); - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); - PostgresTable table = Randomly.fromList(tables); - tables.remove(table); - PostgresJoinType options = PostgresJoinType.getRandom(); - PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); - joinStatements.add(j); - } - // JOIN subqueries - for (int i = 0; i < Randomly.smallNumber(); i++) { - PostgresTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); - PostgresSubquery subquery = PostgresTLPBase.createSubquery(globalState, String.format("sub%d", i), - subqueryTables); - PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); - PostgresJoinType options = PostgresJoinType.getRandom(); - PostgresJoin j = new PostgresJoin(subquery, joinClause, options); - joinStatements.add(j); - } - return joinStatements; - } -} diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java index 77c4fb022..55b5d2dcf 100644 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java +++ b/src/sqlancer/postgres/oracle/tlp/PostgresTLPBase.java @@ -20,13 +20,13 @@ import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresJoin; +import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresSelect; import sqlancer.postgres.ast.PostgresSelect.ForClause; import sqlancer.postgres.ast.PostgresSelect.PostgresFromTable; import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; -import sqlancer.postgres.oracle.PostgresNoRECOracle; public class PostgresTLPBase extends TernaryLogicPartitioningOracleBase implements TestOracle { @@ -53,8 +53,27 @@ public void check() throws SQLException { protected List getJoinStatements(PostgresGlobalState globalState, List columns, List tables) { - return PostgresNoRECOracle.getJoinStatements(state, columns, tables); - // TODO joins + List joinStatements = new ArrayList<>(); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState).setColumns(columns); + for (int i = 1; i < tables.size(); i++) { + PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); + PostgresTable table = Randomly.fromList(tables); + tables.remove(table); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(new PostgresFromTable(table, Randomly.getBoolean()), joinClause, options); + joinStatements.add(j); + } + // JOIN subqueries + for (int i = 0; i < Randomly.smallNumber(); i++) { + PostgresTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); + PostgresSubquery subquery = PostgresTLPBase.createSubquery(globalState, String.format("sub%d", i), + subqueryTables); + PostgresExpression joinClause = gen.generateExpression(PostgresDataType.BOOLEAN); + PostgresJoinType options = PostgresJoinType.getRandom(); + PostgresJoin j = new PostgresJoin(subquery, joinClause, options); + joinStatements.add(j); + } + return joinStatements; } protected void generateSelectBase(List tables, List joins) { diff --git a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java b/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java deleted file mode 100644 index 3c183457d..000000000 --- a/src/sqlancer/postgres/oracle/tlp/PostgresTLPWhereOracle.java +++ /dev/null @@ -1,40 +0,0 @@ -package sqlancer.postgres.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresTable; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; - -public class PostgresTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public PostgresTLPWhereOracle(PostgresGlobalState state) { - PostgresExpressionGenerator gen = new PostgresExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) - .with(PostgresCommon.getCommonFetchErrors()).withRegex(PostgresCommon.getCommonExpressionRegexErrors()) - .build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/presto/PrestoOptions.java b/src/sqlancer/presto/PrestoOptions.java index 78deab27c..f0c557ca5 100644 --- a/src/sqlancer/presto/PrestoOptions.java +++ b/src/sqlancer/presto/PrestoOptions.java @@ -1,24 +1,14 @@ package sqlancer.presto; -import java.util.ArrayList; import java.util.List; import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.presto.test.PrestoNoRECOracle; -import sqlancer.presto.test.PrestoQueryPartitioningAggregateTester; -import sqlancer.presto.test.PrestoQueryPartitioningDistinctTester; -import sqlancer.presto.test.PrestoQueryPartitioningGroupByTester; -import sqlancer.presto.test.PrestoQueryPartitioningHavingTester; -import sqlancer.presto.test.PrestoQueryPartitioningWhereTester; @Parameters(commandDescription = "Presto") -public class PrestoOptions implements DBMSSpecificOptions { +public class PrestoOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 8080; @@ -109,58 +99,4 @@ public List getTestOracleFactory() { return oracles; } - public enum PrestoOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoNoRECOracle(globalState); - } - - }, - HAVING { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningHavingTester(globalState); - } - }, - WHERE { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningWhereTester(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningGroupByTester(globalState); - } - }, - AGGREGATE { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningAggregateTester(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(PrestoGlobalState globalState) { - return new PrestoQueryPartitioningDistinctTester(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(PrestoGlobalState globalState) { - List> oracles = new ArrayList<>(); - oracles.add(new PrestoQueryPartitioningWhereTester(globalState)); - oracles.add(new PrestoQueryPartitioningHavingTester(globalState)); - oracles.add(new PrestoQueryPartitioningAggregateTester(globalState)); - oracles.add(new PrestoQueryPartitioningDistinctTester(globalState)); - oracles.add(new PrestoQueryPartitioningGroupByTester(globalState)); - return new CompositeTestOracle<>(oracles, globalState); - } - } - - } - } diff --git a/src/sqlancer/presto/PrestoOracleFactory.java b/src/sqlancer/presto/PrestoOracleFactory.java new file mode 100644 index 000000000..3076357f2 --- /dev/null +++ b/src/sqlancer/presto/PrestoOracleFactory.java @@ -0,0 +1,73 @@ +package sqlancer.presto; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.presto.gen.PrestoTypedExpressionGenerator; +import sqlancer.presto.test.PrestoQueryPartitioningAggregateTester; +import sqlancer.presto.test.PrestoQueryPartitioningDistinctTester; +import sqlancer.presto.test.PrestoQueryPartitioningGroupByTester; +import sqlancer.presto.test.PrestoQueryPartitioningHavingTester; +import sqlancer.presto.test.PrestoQueryPartitioningWhereTester; + +public enum PrestoOracleFactory implements OracleFactory { + NOREC { + @Override + public TestOracle create(PrestoGlobalState globalState) { + PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) + .with("canceling statement due to statement timeout").build(); + return new NoRECOracle<>(globalState, gen, errors); + } + + }, + HAVING { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningHavingTester(globalState); + } + }, + WHERE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningWhereTester(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningGroupByTester(globalState); + } + }, + AGGREGATE { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningAggregateTester(globalState); + } + + }, + DISTINCT { + @Override + public TestOracle create(PrestoGlobalState globalState) { + return new PrestoQueryPartitioningDistinctTester(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(PrestoGlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + return new CompositeTestOracle<>(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/presto/test/PrestoNoRECOracle.java b/src/sqlancer/presto/test/PrestoNoRECOracle.java deleted file mode 100644 index 6ef5e68ce..000000000 --- a/src/sqlancer/presto/test/PrestoNoRECOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.presto.test; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.presto.PrestoErrors; -import sqlancer.presto.PrestoGlobalState; -import sqlancer.presto.PrestoSchema; -import sqlancer.presto.PrestoSchema.PrestoColumn; -import sqlancer.presto.PrestoSchema.PrestoTable; -import sqlancer.presto.ast.PrestoExpression; -import sqlancer.presto.ast.PrestoJoin; -import sqlancer.presto.ast.PrestoSelect; -import sqlancer.presto.gen.PrestoTypedExpressionGenerator; - -public class PrestoNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public PrestoNoRECOracle(PrestoGlobalState globalState) { - PrestoTypedExpressionGenerator gen = new PrestoTypedExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(PrestoErrors.getExpressionErrors()) - .with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/questdb/QuestDBOptions.java b/src/sqlancer/questdb/QuestDBOptions.java index f84ca3d47..82cf55bd9 100644 --- a/src/sqlancer/questdb/QuestDBOptions.java +++ b/src/sqlancer/questdb/QuestDBOptions.java @@ -1,6 +1,5 @@ package sqlancer.questdb; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,11 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.questdb.QuestDBOptions.QuestDBOracleFactory; -import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; -import sqlancer.questdb.test.QuestDBQueryPartitioningWhereTester; @Parameters(separators = "=", commandDescription = "QuestDB (default port: " + QuestDBOptions.DEFAULT_PORT + " default host: " + QuestDBOptions.DEFAULT_HOST + ")") @@ -23,16 +17,6 @@ public class QuestDBOptions implements DBMSSpecificOptions @Parameter(names = "--oracle") public List oracle = Arrays.asList(QuestDBOracleFactory.WHERE); - public enum QuestDBOracleFactory implements OracleFactory { - // TODO (anxing): implement test oracles - WHERE { - @Override - public TestOracle create(QuestDBGlobalState globalState) throws SQLException { - return new QuestDBQueryPartitioningWhereTester(globalState); - } - } - } - @Parameter(names = "--username", description = "The user name used to log into QuestDB") private String userName = "admin"; // NOPMD diff --git a/src/sqlancer/questdb/QuestDBOracleFactory.java b/src/sqlancer/questdb/QuestDBOracleFactory.java new file mode 100644 index 000000000..52c727278 --- /dev/null +++ b/src/sqlancer/questdb/QuestDBOracleFactory.java @@ -0,0 +1,18 @@ +package sqlancer.questdb; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.questdb.test.QuestDBQueryPartitioningWhereTester; + +public enum QuestDBOracleFactory implements OracleFactory { + // TODO (anxing): implement test oracles + WHERE { + @Override + public TestOracle create(QuestDBProvider.QuestDBGlobalState globalState) + throws SQLException { + return new QuestDBQueryPartitioningWhereTester(globalState); + } + } +} diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index e2c04ed46..ec90f3bca 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -1,7 +1,5 @@ package sqlancer.sqlite3; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,18 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; -import sqlancer.sqlite3.oracle.SQLite3Fuzzer; -import sqlancer.sqlite3.oracle.SQLite3NoRECOracle; -import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPAggregateOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPDistinctOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPGroupByOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPHavingOracle; -import sqlancer.sqlite3.oracle.tlp.SQLite3TLPWhereOracle; @Parameters(separators = "=", commandDescription = "SQLite3") public class SQLite3Options implements DBMSSpecificOptions { @@ -93,80 +79,6 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") public int maxNumIndexes = 20; - public enum SQLite3OracleFactory implements OracleFactory { - PQS { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3PivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - - }, - NoREC { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3NoRECOracle(globalState); - } - }, - AGGREGATE { - - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPAggregateOracle(globalState); - } - - }, - WHERE { - - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPWhereOracle(globalState); - } - - }, - DISTINCT { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPDistinctOracle(globalState); - } - }, - GROUP_BY { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPGroupByOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3TLPHavingOracle(globalState); - } - }, - FUZZER { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - return new SQLite3Fuzzer(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(SQLite3GlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new SQLite3TLPWhereOracle(globalState)); - oracles.add(new SQLite3TLPDistinctOracle(globalState)); - oracles.add(new SQLite3TLPGroupByOracle(globalState)); - oracles.add(new SQLite3TLPHavingOracle(globalState)); - oracles.add(new SQLite3TLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return Arrays.asList(oracles); diff --git a/src/sqlancer/sqlite3/SQLite3OracleFactory.java b/src/sqlancer/sqlite3/SQLite3OracleFactory.java new file mode 100644 index 000000000..b1a9a1265 --- /dev/null +++ b/src/sqlancer/sqlite3/SQLite3OracleFactory.java @@ -0,0 +1,101 @@ +package sqlancer.sqlite3; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.oracle.SQLite3Fuzzer; +import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPAggregateOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPDistinctOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPGroupByOracle; +import sqlancer.sqlite3.oracle.tlp.SQLite3TLPHavingOracle; + +public enum SQLite3OracleFactory implements OracleFactory { + PQS { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3PivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + + }, + NoREC { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .with(SQLite3Errors.getMatchQueryErrors()).with(SQLite3Errors.getQueryErrors()) + .with("misuse of aggregate", "misuse of window function", + "second argument to nth_value must be a positive integer", "no such table", + "no query solution", "unable to use function MATCH in the requested context") + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + AGGREGATE { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPAggregateOracle(globalState); + } + + }, + WHERE { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) + .build(); + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + + }, + DISTINCT { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPDistinctOracle(globalState); + } + }, + GROUP_BY { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPGroupByOracle(globalState); + } + }, + HAVING { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3TLPHavingOracle(globalState); + } + }, + FUZZER { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3Fuzzer(globalState); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(DISTINCT.create(globalState)); + oracles.add(GROUP_BY.create(globalState)); + oracles.add(HAVING.create(globalState)); + oracles.add(AGGREGATE.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }; + +} diff --git a/src/sqlancer/sqlite3/SQLite3Provider.java b/src/sqlancer/sqlite3/SQLite3Provider.java index b20701c95..5fbd3b471 100644 --- a/src/sqlancer/sqlite3/SQLite3Provider.java +++ b/src/sqlancer/sqlite3/SQLite3Provider.java @@ -23,7 +23,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; import sqlancer.sqlite3.gen.SQLite3AnalyzeGenerator; import sqlancer.sqlite3.gen.SQLite3CreateVirtualRtreeTabelGenerator; import sqlancer.sqlite3.gen.SQLite3ExplainGenerator; diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index e07533ed8..1d045982b 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; +import sqlancer.sqlite3.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java index 6c27a5d58..7847208a3 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3TableGenerator.java @@ -12,7 +12,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; +import sqlancer.sqlite3.SQLite3OracleFactory; import sqlancer.sqlite3.gen.SQLite3ColumnBuilder; import sqlancer.sqlite3.gen.SQLite3Common; import sqlancer.sqlite3.schema.SQLite3Schema; diff --git a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java index 1c483e59d..bc605a015 100644 --- a/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java +++ b/src/sqlancer/sqlite3/gen/ddl/SQLite3ViewGenerator.java @@ -9,7 +9,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.SQLite3Options.SQLite3OracleFactory; +import sqlancer.sqlite3.SQLite3OracleFactory; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Select; diff --git a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java deleted file mode 100644 index 8f6e2d276..000000000 --- a/src/sqlancer/sqlite3/oracle/SQLite3NoRECOracle.java +++ /dev/null @@ -1,48 +0,0 @@ -package sqlancer.sqlite3.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.ast.SQLite3Expression.Join; -import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; -import sqlancer.sqlite3.schema.SQLite3Schema; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; - -public class SQLite3NoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public SQLite3NoRECOracle(SQLite3GlobalState globalState) { - SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) - .with(SQLite3Errors.getMatchQueryErrors()).with(SQLite3Errors.getQueryErrors()) - .with("misuse of aggregate", "misuse of window function", - "second argument to nth_value must be a positive integer", "no such table", "no query solution", - "unable to use function MATCH in the requested context") - .build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java b/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java deleted file mode 100644 index 8d850fdcb..000000000 --- a/src/sqlancer/sqlite3/oracle/tlp/SQLite3TLPWhereOracle.java +++ /dev/null @@ -1,37 +0,0 @@ -package sqlancer.sqlite3.oracle.tlp; - -import java.sql.SQLException; - -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.sqlite3.SQLite3Errors; -import sqlancer.sqlite3.SQLite3GlobalState; -import sqlancer.sqlite3.ast.SQLite3Expression; -import sqlancer.sqlite3.ast.SQLite3Select; -import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; -import sqlancer.sqlite3.schema.SQLite3Schema; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; -import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; - -public class SQLite3TLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public SQLite3TLPWhereOracle(SQLite3GlobalState state) { - SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SQLite3Errors.getExpectedExpressionErrors()) - .build(); - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 6c69e84a1..1619832d6 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -1,7 +1,5 @@ package sqlancer.tidb; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,15 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; -import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.oracle.TiDBCERTOracle; -import sqlancer.tidb.oracle.TiDBDQPOracle; -import sqlancer.tidb.oracle.TiDBTLPHavingOracle; -import sqlancer.tidb.oracle.TiDBTLPWhereOracle; @Parameters(separators = "=", commandDescription = "TiDB (default port: " + TiDBOptions.DEFAULT_PORT + ", default host: " + TiDBOptions.DEFAULT_HOST + ")") @@ -40,48 +29,6 @@ public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = { "--tiflash" }, description = "Enable TiFlash") public boolean tiflash; - public enum TiDBOracleFactory implements OracleFactory { - HAVING { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBTLPHavingOracle(globalState); - } - }, - WHERE { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBTLPWhereOracle(globalState); - } - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new TiDBTLPWhereOracle(globalState)); - oracles.add(new TiDBTLPHavingOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - }, - CERT { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBCERTOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - DQP { - @Override - public TestOracle create(TiDBGlobalState globalState) throws SQLException { - return new TiDBDQPOracle(globalState); - } - }; - - } - @Override public List getTestOracleFactory() { return oracle; diff --git a/src/sqlancer/tidb/TiDBOracleFactory.java b/src/sqlancer/tidb/TiDBOracleFactory.java new file mode 100644 index 000000000..fac172adf --- /dev/null +++ b/src/sqlancer/tidb/TiDBOracleFactory.java @@ -0,0 +1,64 @@ +package sqlancer.tidb; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.tidb.oracle.TiDBCERTOracle; +import sqlancer.tidb.oracle.TiDBDQPOracle; +import sqlancer.tidb.oracle.TiDBTLPHavingOracle; + +public enum TiDBOracleFactory implements OracleFactory { + HAVING { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + return new TiDBTLPHavingOracle(globalState); + } + }, + WHERE { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws Exception { + List> oracles = new ArrayList<>(); + oracles.add(WHERE.create(globalState)); + oracles.add(HAVING.create(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + }, + CERT { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + return new TiDBCERTOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + DQP { + @Override + public TestOracle create(TiDBProvider.TiDBGlobalState globalState) + throws SQLException { + return new TiDBDQPOracle(globalState); + } + }; + +} diff --git a/src/sqlancer/tidb/TiDBProvider.java b/src/sqlancer/tidb/TiDBProvider.java index 05ef0d9c6..1d117e234 100644 --- a/src/sqlancer/tidb/TiDBProvider.java +++ b/src/sqlancer/tidb/TiDBProvider.java @@ -23,7 +23,6 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.tidb.TiDBOptions.TiDBOracleFactory; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.gen.TiDBAlterTableGenerator; diff --git a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java b/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java deleted file mode 100644 index 61ec16788..000000000 --- a/src/sqlancer/tidb/oracle/TiDBTLPWhereOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.tidb.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.TLPWhereOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.tidb.TiDBErrors; -import sqlancer.tidb.TiDBExpressionGenerator; -import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.TiDBSchema; -import sqlancer.tidb.TiDBSchema.TiDBColumn; -import sqlancer.tidb.TiDBSchema.TiDBTable; -import sqlancer.tidb.ast.TiDBExpression; -import sqlancer.tidb.ast.TiDBJoin; -import sqlancer.tidb.ast.TiDBSelect; - -public class TiDBTLPWhereOracle implements TestOracle { - - private final TLPWhereOracle oracle; - - public TiDBTLPWhereOracle(TiDBGlobalState state) { - TiDBExpressionGenerator gen = new TiDBExpressionGenerator(state); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); - - this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } -} diff --git a/src/sqlancer/yugabyte/ycql/YCQLOptions.java b/src/sqlancer/yugabyte/ycql/YCQLOptions.java index df4854757..9d865aca2 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLOptions.java +++ b/src/sqlancer/yugabyte/ycql/YCQLOptions.java @@ -1,6 +1,5 @@ package sqlancer.yugabyte.ycql; -import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -8,11 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.TestOracle; -import sqlancer.yugabyte.ycql.YCQLOptions.YCQLOracleFactory; -import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; -import sqlancer.yugabyte.ycql.test.YCQLFuzzer; @Parameters(separators = "=", commandDescription = "YCQL (default port: " + YCQLOptions.DEFAULT_PORT + ", default host: " + YCQLOptions.DEFAULT_HOST) @@ -34,16 +28,6 @@ public class YCQLOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracles = Arrays.asList(YCQLOracleFactory.FUZZER); - public enum YCQLOracleFactory implements OracleFactory { - FUZZER { - @Override - public TestOracle create(YCQLGlobalState globalState) throws SQLException { - return new YCQLFuzzer(globalState); - } - - } - } - @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java b/src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java new file mode 100644 index 000000000..beaf43704 --- /dev/null +++ b/src/sqlancer/yugabyte/ycql/YCQLOracleFactory.java @@ -0,0 +1,18 @@ +package sqlancer.yugabyte.ycql; + +import java.sql.SQLException; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TestOracle; +import sqlancer.yugabyte.ycql.test.YCQLFuzzer; + +public enum YCQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YCQLProvider.YCQLGlobalState globalState) + throws SQLException { + return new YCQLFuzzer(globalState); + } + + } +} diff --git a/src/sqlancer/yugabyte/ysql/YSQLOptions.java b/src/sqlancer/yugabyte/ysql/YSQLOptions.java index 6b46b09bf..71b886ba5 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLOptions.java +++ b/src/sqlancer/yugabyte/ysql/YSQLOptions.java @@ -1,7 +1,5 @@ package sqlancer.yugabyte.ysql; -import java.sql.SQLException; -import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -9,17 +7,6 @@ import com.beust.jcommander.Parameters; import sqlancer.DBMSSpecificOptions; -import sqlancer.OracleFactory; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.yugabyte.ysql.YSQLOptions.YSQLOracleFactory; -import sqlancer.yugabyte.ysql.oracle.YSQLCatalog; -import sqlancer.yugabyte.ysql.oracle.YSQLFuzzer; -import sqlancer.yugabyte.ysql.oracle.YSQLNoRECOracle; -import sqlancer.yugabyte.ysql.oracle.YSQLPivotedQuerySynthesisOracle; -import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPAggregateOracle; -import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPHavingOracle; -import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPWhereOracle; @Parameters(separators = "=", commandDescription = "YSQL (default port: " + YSQLOptions.DEFAULT_PORT + ", default host: " + YSQLOptions.DEFAULT_HOST) @@ -45,54 +32,4 @@ public List getTestOracleFactory() { return oracle; } - public enum YSQLOracleFactory implements OracleFactory { - FUZZER { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLFuzzer(globalState); - } - }, - CATALOG { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLCatalog(globalState); - } - }, - NOREC { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLNoRECOracle(globalState); - } - }, - PQS { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLPivotedQuerySynthesisOracle(globalState); - } - - @Override - public boolean requiresAllTablesToContainRows() { - return true; - } - }, - HAVING { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - return new YSQLTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(YSQLGlobalState globalState) throws SQLException { - List> oracles = new ArrayList<>(); - oracles.add(new YSQLTLPWhereOracle(globalState)); - oracles.add(new YSQLTLPHavingOracle(globalState)); - oracles.add(new YSQLTLPAggregateOracle(globalState)); - return new CompositeTestOracle(oracles, globalState); - } - } - - } - } diff --git a/src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java b/src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java new file mode 100644 index 000000000..b85f80593 --- /dev/null +++ b/src/sqlancer/yugabyte/ysql/YSQLOracleFactory.java @@ -0,0 +1,72 @@ +package sqlancer.yugabyte.ysql; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.OracleFactory; +import sqlancer.common.oracle.CompositeTestOracle; +import sqlancer.common.oracle.NoRECOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; +import sqlancer.yugabyte.ysql.oracle.YSQLCatalog; +import sqlancer.yugabyte.ysql.oracle.YSQLFuzzer; +import sqlancer.yugabyte.ysql.oracle.YSQLPivotedQuerySynthesisOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPAggregateOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPHavingOracle; +import sqlancer.yugabyte.ysql.oracle.tlp.YSQLTLPWhereOracle; + +public enum YSQLOracleFactory implements OracleFactory { + FUZZER { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLFuzzer(globalState); + } + }, + CATALOG { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLCatalog(globalState); + } + }, + NOREC { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) + .with(YSQLErrors.getCommonFetchErrors()).with("canceling statement due to statement timeout") + .build(); + return new NoRECOracle<>(globalState, gen, errors); + } + }, + PQS { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLPivotedQuerySynthesisOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } + }, + HAVING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + return new YSQLTLPHavingOracle(globalState); + } + + }, + QUERY_PARTITIONING { + @Override + public TestOracle create(YSQLGlobalState globalState) throws SQLException { + List> oracles = new ArrayList<>(); + oracles.add(new YSQLTLPWhereOracle(globalState)); + oracles.add(new YSQLTLPHavingOracle(globalState)); + oracles.add(new YSQLTLPAggregateOracle(globalState)); + return new CompositeTestOracle(oracles, globalState); + } + } + +} diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java deleted file mode 100644 index 70804ce40..000000000 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLNoRECOracle.java +++ /dev/null @@ -1,44 +0,0 @@ -package sqlancer.yugabyte.ysql.oracle; - -import java.sql.SQLException; - -import sqlancer.Reproducer; -import sqlancer.common.oracle.NoRECOracle; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.yugabyte.ysql.YSQLErrors; -import sqlancer.yugabyte.ysql.YSQLGlobalState; -import sqlancer.yugabyte.ysql.YSQLSchema; -import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; -import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; -import sqlancer.yugabyte.ysql.ast.YSQLExpression; -import sqlancer.yugabyte.ysql.ast.YSQLJoin; -import sqlancer.yugabyte.ysql.ast.YSQLSelect; -import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; - -public class YSQLNoRECOracle implements TestOracle { - - NoRECOracle oracle; - - public YSQLNoRECOracle(YSQLGlobalState globalState) { - YSQLExpressionGenerator gen = new YSQLExpressionGenerator(globalState); - ExpectedErrors errors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) - .with(YSQLErrors.getCommonFetchErrors()).with("canceling statement due to statement timeout").build(); - this.oracle = new NoRECOracle<>(globalState, gen, errors); - } - - @Override - public void check() throws SQLException { - oracle.check(); - } - - @Override - public Reproducer getLastReproducer() { - return oracle.getLastReproducer(); - } - - @Override - public String getLastQueryString() { - return oracle.getLastQueryString(); - } -} From aed8cc1c317e31fe26a4643276b44126f1825ac7 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 25 Sep 2024 12:09:10 +0800 Subject: [PATCH 574/856] [YSQL] Implement TLP-where expression generator interface --- .../ysql/gen/YSQLExpressionGenerator.java | 22 ++++++++++++++----- .../ysql/gen/YSQLRandomQueryGenerator.java | 2 +- .../YSQLPivotedQuerySynthesisOracle.java | 2 +- .../oracle/tlp/YSQLTLPAggregateOracle.java | 2 +- .../ysql/oracle/tlp/YSQLTLPHavingOracle.java | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java index 759d4d3ba..80c35c431 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLExpressionGenerator.java @@ -12,6 +12,7 @@ import sqlancer.Randomly; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.yugabyte.ysql.YSQLCompoundDataType; import sqlancer.yugabyte.ysql.YSQLGlobalState; @@ -47,7 +48,8 @@ import sqlancer.yugabyte.ysql.ast.YSQLSimilarTo; public class YSQLExpressionGenerator implements ExpressionGenerator, - NoRECGenerator { + NoRECGenerator, + TLPWhereGenerator { private final int maxDepth; @@ -182,7 +184,8 @@ public YSQLExpression generateExpression(int depth) { return generateExpression(depth, YSQLDataType.getRandomType()); } - public List generateOrderBy() { + @Override + public List generateOrderBys() { List orderBys = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber(); i++) { orderBys.add(new YSQLOrderByTerm(YSQLColumnValue.create(Randomly.fromList(columns), null), @@ -584,7 +587,7 @@ public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState select.setWhereClause(gen.generateExpression(0, YSQLDataType.BOOLEAN)); } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); @@ -599,8 +602,7 @@ public static YSQLSelect.YSQLSubquery createSubquery(YSQLGlobalState globalState } @Override - public NoRECGenerator setTablesAndColumns( - AbstractTables tables) { + public YSQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -658,7 +660,7 @@ public String generateOptimizedQueryString(YSQLSelect select, YSQLExpression whe YSQLColumnValue allColumns = new YSQLColumnValue(Randomly.fromList(columns), null); select.setFetchColumns(Arrays.asList(allColumns)); if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(generateOrderBy()); + select.setOrderByClauses(generateOrderBys()); } select.setWhereClause(whereCondition); } @@ -676,4 +678,12 @@ public String generateUnoptimizedQueryString(YSQLSelect select, YSQLExpression w return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public List generateFetchColumns(boolean shouldCreateDummy) { + if (shouldCreateDummy && Randomly.getBooleanWithRatherLowProbability()) { + return List.of(new YSQLColumnValue(YSQLColumn.createDummy("*"), null)); + } + return columns.stream().map(c -> new YSQLColumnValue(c, null)).collect(Collectors.toList()); + } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java index 02e87dffd..9821dcb7d 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLRandomQueryGenerator.java @@ -45,7 +45,7 @@ public static YSQLSelect createRandomQuery(int nrColumns, YSQLGlobalState global } } if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { select.setLimitClause(YSQLConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); diff --git a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java index ac321d7c3..8f54aa698 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/YSQLPivotedQuerySynthesisOracle.java @@ -134,7 +134,7 @@ public SQLQueryAdapter getRectifiedQuery() throws SQLException { YSQLExpression offsetClause = generateOffset(); selectStatement.setOffsetClause(offsetClause); } - List orderBy = new YSQLExpressionGenerator(globalState).setColumns(columns).generateOrderBy(); + List orderBy = new YSQLExpressionGenerator(globalState).setColumns(columns).generateOrderBys(); selectStatement.setOrderByClauses(orderBy); return new SQLQueryAdapter(YSQLVisitor.asString(selectStatement)); } diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java index 6e9b74f87..3fdaf3f9a 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPAggregateOracle.java @@ -61,7 +61,7 @@ protected void aggregateCheck() throws SQLException { } select.setFetchColumns(Arrays.asList(aggregate)); if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } originalQuery = YSQLVisitor.asString(select); firstResult = getAggregateResult(originalQuery); diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java index d4a398cf2..32df70b11 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPHavingOracle.java @@ -43,7 +43,7 @@ protected void havingCheck() throws SQLException { boolean orderBy = Randomly.getBoolean(); if (orderBy) { - select.setOrderByClauses(gen.generateOrderBy()); + select.setOrderByClauses(gen.generateOrderBys()); } select.setHavingClause(predicate); String firstQueryString = YSQLVisitor.asString(select); From 510df57e895e16ced536a0466e1e8014e7a95130 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 25 Sep 2024 12:09:28 +0800 Subject: [PATCH 575/856] [YSQL] Use common TLP-where oracle --- .../ysql/oracle/tlp/YSQLTLPWhereOracle.java | 58 +++++++++---------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java index 0d61dd8c5..265586586 100644 --- a/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java +++ b/src/sqlancer/yugabyte/ysql/oracle/tlp/YSQLTLPWhereOracle.java @@ -1,45 +1,45 @@ package sqlancer.yugabyte.ysql.oracle.tlp; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; -import sqlancer.yugabyte.ysql.YSQLVisitor; +import sqlancer.yugabyte.ysql.YSQLSchema; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLColumn; +import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; +import sqlancer.yugabyte.ysql.ast.YSQLExpression; +import sqlancer.yugabyte.ysql.ast.YSQLJoin; +import sqlancer.yugabyte.ysql.ast.YSQLSelect; +import sqlancer.yugabyte.ysql.gen.YSQLExpressionGenerator; -public class YSQLTLPWhereOracle extends YSQLTLPBase { +public class YSQLTLPWhereOracle implements TestOracle { + + private final TLPWhereOracle oracle; public YSQLTLPWhereOracle(YSQLGlobalState state) { - super(state); + YSQLExpressionGenerator gen = new YSQLExpressionGenerator(state); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(YSQLErrors.getCommonExpressionErrors()) + .with(YSQLErrors.getCommonFetchErrors()).build(); + + this.oracle = new TLPWhereOracle<>(state, gen, expectedErrors); } @Override public void check() throws SQLException { - super.check(); - whereCheck(); + oracle.check(); + } + + @Override + public String getLastQueryString() { + return oracle.getLastQueryString(); } - protected void whereCheck() throws SQLException { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - String originalQueryString = YSQLVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = YSQLVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = YSQLVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = YSQLVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); + @Override + public Reproducer getLastReproducer() { + return oracle.getLastReproducer(); } } From e0d072bec939074cc2ee0ab0ddaded5d563fd075 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 1 Oct 2024 16:12:46 +0800 Subject: [PATCH 576/856] [Postgres] Add CERT test --- .github/workflows/main.yml | 1 + test/sqlancer/dbms/TestPostgresCERT.java | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 test/sqlancer/dbms/TestPostgresCERT.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68ed03dc6..2387644fd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -482,6 +482,7 @@ jobs: POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresPQS test POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresTLP test POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresNoREC test + POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresCERT test presto: name: DBMS Tests (Presto) diff --git a/test/sqlancer/dbms/TestPostgresCERT.java b/test/sqlancer/dbms/TestPostgresCERT.java new file mode 100644 index 000000000..afa5e3801 --- /dev/null +++ b/test/sqlancer/dbms/TestPostgresCERT.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestPostgresCERT { + + @Test + public void testCERT() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.POSTGRES_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "postgres", "--test-collations", + "false", "--oracle", "CERT" })); + } +} From eefcde54c4ec9df85ce0d20c934cdc93067a7ae8 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 1 Oct 2024 16:26:43 +0800 Subject: [PATCH 577/856] [Postgres] Add bug 18643 --- src/sqlancer/postgres/PostgresBugs.java | 10 ++++++++++ .../postgres/oracle/PostgresCERTOracle.java | 18 ++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 src/sqlancer/postgres/PostgresBugs.java diff --git a/src/sqlancer/postgres/PostgresBugs.java b/src/sqlancer/postgres/PostgresBugs.java new file mode 100644 index 000000000..352d6c1c5 --- /dev/null +++ b/src/sqlancer/postgres/PostgresBugs.java @@ -0,0 +1,10 @@ +package sqlancer.postgres; + +// do not make the fields final to avoid warnings +public final class PostgresBugs { + public static boolean bug18643 = true; + + private PostgresBugs() { + } + +} diff --git a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java index 9b509834e..d22689816 100644 --- a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java +++ b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java @@ -14,6 +14,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.postgres.PostgresBugs; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -77,14 +78,19 @@ public void check() throws SQLException { // First query row count String queryString1 = PostgresVisitor.asString(select); - int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + long rowCount1 = getRow(state, queryString1, queryPlan1Sequences); // JOIN and LIMIT mutations not added - boolean increase = mutate(Mutator.LIMIT); + boolean increase; + if (PostgresBugs.bug18643) { + increase = mutate(Mutator.LIMIT, Mutator.OR, Mutator.AND); + } else { + increase = mutate(Mutator.LIMIT); + } // Second Query row count String queryString2 = PostgresVisitor.asString(select); - int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + long rowCount2 = getRow(state, queryString2, queryPlan2Sequences); // Check query plan equivalence if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { @@ -216,9 +222,9 @@ protected boolean mutateLimit() { return increase; } - private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) + private long getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) throws AssertionError, SQLException { - int row = -1; + long row = -1; String explainQuery = "EXPLAIN " + selectStr; if (globalState.getOptions().logEachSelect()) { @@ -239,7 +245,7 @@ private int getRow(SQLGlobalState globalState, String selectStr, List Date: Wed, 2 Oct 2024 11:55:51 +0800 Subject: [PATCH 578/856] [TiDB] Add CERT test --- .github/workflows/main.yml | 4 +++- test/sqlancer/dbms/TestTiDBCERT.java | 19 +++++++++++++++++++ .../dbms/{TestTiDB.java => TestTiDBTLP.java} | 8 +++----- 3 files changed, 25 insertions(+), 6 deletions(-) create mode 100644 test/sqlancer/dbms/TestTiDBCERT.java rename test/sqlancer/dbms/{TestTiDB.java => TestTiDBTLP.java} (63%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68ed03dc6..2e2dae97e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -574,7 +574,9 @@ jobs: - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests - run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDB test + run: | + TIDB_AVAILABLE=true mvn -Dtest=TestTiDBTLP test + TIDB_AVAILABLE=true mvn -Dtest=TestTiDBCERT test tidb-qpg: name: QPG Tests (TiDB) diff --git a/test/sqlancer/dbms/TestTiDBCERT.java b/test/sqlancer/dbms/TestTiDBCERT.java new file mode 100644 index 000000000..748444870 --- /dev/null +++ b/test/sqlancer/dbms/TestTiDBCERT.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestTiDBCERT { + + @Test + public void testCERT() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.TIDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-queries", "4", "tidb", "--oracle", "CERT" })); + } + +} diff --git a/test/sqlancer/dbms/TestTiDB.java b/test/sqlancer/dbms/TestTiDBTLP.java similarity index 63% rename from test/sqlancer/dbms/TestTiDB.java rename to test/sqlancer/dbms/TestTiDBTLP.java index b97e59273..11cf7581e 100644 --- a/test/sqlancer/dbms/TestTiDB.java +++ b/test/sqlancer/dbms/TestTiDBTLP.java @@ -7,13 +7,11 @@ import sqlancer.Main; -public class TestTiDB { +public class TestTiDBTLP { @Test - public void testMySQL() { - String tiDB = System.getenv("TIDB_AVAILABLE"); - boolean tiDBIsAvailable = tiDB != null && tiDB.equalsIgnoreCase("true"); - assumeTrue(tiDBIsAvailable); + public void testTLP() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.TIDB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "tidb" })); } From 648bc945b2bdf2b169740aa7b950d211d9f6b2b2 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 2 Oct 2024 11:56:25 +0800 Subject: [PATCH 579/856] [TiDB] Handle expected errors --- src/sqlancer/tidb/TiDBSchema.java | 2 ++ src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java | 3 ++- src/sqlancer/tidb/gen/TiDBDeleteGenerator.java | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 265cacbda..7c808beee 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -251,6 +251,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { primitiveType = TiDBDataType.INT; size = 1; break; + case "tinyint(2)": case "tinyint(3)": case "tinyint(4)": primitiveType = TiDBDataType.INT; @@ -267,6 +268,7 @@ private static TiDBCompositeDataType getColumnType(String typeString) { size = 4; break; case "blob": + case "mediumblob": case "longblob": case "tinyblob": primitiveType = TiDBDataType.BLOB; diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index f78878ac0..8b43c635a 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -5,6 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -14,7 +15,7 @@ private TiDBAnalyzeTableGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - ExpectedErrors errors = new ExpectedErrors(); + ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); StringBuilder sb = new StringBuilder("ANALYZE TABLE "); diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index f6eea4671..c3986f8d9 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -18,7 +18,7 @@ private TiDBDeleteGenerator() { } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - ExpectedErrors errors = new ExpectedErrors(); + ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); StringBuilder sb = new StringBuilder("DELETE "); From bd45822bc8d6083a0bdb6569f91d625de6fba074 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 2 Oct 2024 11:56:54 +0800 Subject: [PATCH 580/856] [TiDB] Add bugs 38319 and 51525 --- src/sqlancer/tidb/TiDBBugs.java | 6 ++++++ src/sqlancer/tidb/gen/TiDBViewGenerator.java | 8 +++++++- src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 11 ++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/tidb/TiDBBugs.java b/src/sqlancer/tidb/TiDBBugs.java index 81f27b705..3849a3c8c 100644 --- a/src/sqlancer/tidb/TiDBBugs.java +++ b/src/sqlancer/tidb/TiDBBugs.java @@ -14,6 +14,9 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/38295 public static boolean bug38295 = true; + // https://github.com/pingcap/tidb/issues/38319 + public static boolean bug38319 = true; + // https://github.com/pingcap/tidb/issues/44747 public static boolean bug44747 = true; @@ -32,6 +35,9 @@ public final class TiDBBugs { // https://github.com/pingcap/tidb/issues/47348 public static boolean bug47348 = true; + // https://github.com/pingcap/tidb/issues/51525 + public static boolean bug51525 = true; + private TiDBBugs() { } diff --git a/src/sqlancer/tidb/gen/TiDBViewGenerator.java b/src/sqlancer/tidb/gen/TiDBViewGenerator.java index fdbbc0839..79c284ecd 100644 --- a/src/sqlancer/tidb/gen/TiDBViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBViewGenerator.java @@ -4,8 +4,10 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.ast.TiDBSelect; public final class TiDBViewGenerator { @@ -37,7 +39,11 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { sb.append(i); } sb.append(") AS "); - sb.append(TiDBRandomQuerySynthesizer.generate(globalState, nrColumns).getQueryString()); + TiDBSelect select = TiDBRandomQuerySynthesizer.generateSelect(globalState, nrColumns); + if (TiDBBugs.bug38319 && !select.getGroupByExpressions().isEmpty()) { + throw new IgnoreMeException(); + } + sb.append(select.asString()); ExpectedErrors errors = new ExpectedErrors(); TiDBErrors.addExpressionErrors(errors); errors.add( diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java index 4fd5c8c54..b55c1df1f 100644 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java @@ -14,6 +14,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.tidb.TiDBBugs; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; @@ -82,8 +83,16 @@ public void check() throws SQLException { String queryString1 = TiDBVisitor.asString(select); double rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + List excludes = new ArrayList<>(); + excludes.add(Mutator.DISTINCT); + if (TiDBBugs.bug51525) { + excludes.add(Mutator.OR); + } + if (TiDBBugs.bug38319) { + excludes.add(Mutator.GROUPBY); + } // Mutate the query - boolean increase = mutate(Mutator.DISTINCT); + boolean increase = mutate(excludes.toArray(new Mutator[0])); // Get the result of the second query String queryString2 = TiDBVisitor.asString(select); From 880ec6f3f26985ade2217606a591ee9a2cbe4e47 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 1 Oct 2024 14:23:42 +0800 Subject: [PATCH 581/856] [CockroachDB] Add CERT test --- .github/workflows/main.yml | 1 + test/sqlancer/dbms/TestCockroachDBCERT.java | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 test/sqlancer/dbms/TestCockroachDBCERT.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6842b20cc..881ccf689 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -173,6 +173,7 @@ jobs: run: | COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBNoREC test COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBTLP test + COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBCERT test cockroachdb-qpg: name: QPG Tests (CockroachDB) diff --git a/test/sqlancer/dbms/TestCockroachDBCERT.java b/test/sqlancer/dbms/TestCockroachDBCERT.java new file mode 100644 index 000000000..7e60818c1 --- /dev/null +++ b/test/sqlancer/dbms/TestCockroachDBCERT.java @@ -0,0 +1,19 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestCockroachDBCERT { + + @Test + public void testCockroachDBCERT() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "CERT" })); + } + +} From fb3deab1b537402e6bfea33373caa474fafc5d79 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 1 Oct 2024 14:24:19 +0800 Subject: [PATCH 582/856] [CockroachDB] Update CERT oracle --- .../cockroachdb/oracle/CockroachDBCERTOracle.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java index 86d5d75ce..731d6cb14 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -132,11 +132,14 @@ protected boolean mutateJoin() { } JoinType newJoinType = CockroachDBJoin.JoinType.INNER; - if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invariant relation // between LEFT and RIGHT // join - newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); - } else { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS, JoinType.LEFT, + JoinType.RIGHT); + } else if (join.getJoinType() == JoinType.FULL) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS); + } else if (join.getJoinType() != JoinType.CROSS) { newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); } assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT @@ -230,7 +233,7 @@ protected boolean mutateLimit() { private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) throws AssertionError, SQLException { int row = -1; - String explainQuery = "EXPLAIN (VERBOSE) " + selectStr; + String explainQuery = "EXPLAIN " + selectStr; // Log the query if (globalState.getOptions().logEachSelect()) { From 012f4582a97de0119b314d4e35256b394306a270 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Fri, 4 Oct 2024 14:12:14 +0800 Subject: [PATCH 583/856] [CockroachDB] Add bugs 131640, 131647 and 131875 --- src/sqlancer/cockroachdb/CockroachDBBugs.java | 9 +++++++++ .../oracle/CockroachDBCERTOracle.java | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBBugs.java b/src/sqlancer/cockroachdb/CockroachDBBugs.java index 6c34bcac4..86d88ee21 100644 --- a/src/sqlancer/cockroachdb/CockroachDBBugs.java +++ b/src/sqlancer/cockroachdb/CockroachDBBugs.java @@ -59,6 +59,15 @@ public final class CockroachDBBugs { // https://github.com/cockroachdb/cockroach/issues/85230 public static boolean bug85230 = true; + // https://github.com/cockroachdb/cockroach/issues/131640 + public static boolean bug131640 = true; + + // https://github.com/cockroachdb/cockroach/issues/131647 + public static boolean bug131647 = true; + + // https://github.com/cockroachdb/cockroach/issues/131875 + public static boolean bug131875 = true; + private CockroachDBBugs() { } diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java index 731d6cb14..07a51861f 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLGlobalState; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBErrors; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; @@ -76,9 +77,17 @@ public void check() throws SQLException { String queryString1 = CockroachDBVisitor.asString(select); int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); - // Mutate the query + List excludes = new ArrayList<>(); // Disable limit due to its false positive - boolean increase = mutate(); + excludes.add(Mutator.LIMIT); + if (CockroachDBBugs.bug131640) { + excludes.add(Mutator.OR); + } + if (CockroachDBBugs.bug131647) { + excludes.add(Mutator.JOIN); + } + // Mutate the query + boolean increase = mutate(excludes.toArray(new Mutator[0])); // Get the result of the second query String queryString2 = CockroachDBVisitor.asString(select); @@ -262,10 +271,15 @@ private int getRow(SQLGlobalState globalState, String selectStr, List Date: Mon, 7 Oct 2024 09:45:12 +0000 Subject: [PATCH 584/856] Bump org.junit.jupiter:junit-jupiter-engine from 5.9.1 to 5.11.2 Bumps [org.junit.jupiter:junit-jupiter-engine](https://github.com/junit-team/junit5) from 5.9.1 to 5.11.2. - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.9.1...r5.11.2) --- updated-dependencies: - dependency-name: org.junit.jupiter:junit-jupiter-engine dependency-type: direct:development update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 7b97a3a0d..119ef6434 100644 --- a/pom.xml +++ b/pom.xml @@ -324,7 +324,7 @@ org.junit.jupiter junit-jupiter-engine - 5.9.1 + 5.11.2 test From b0f16c9ab63b4178390cc1a711e3792dae6fec26 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 09:55:04 +0800 Subject: [PATCH 585/856] [MySQL] Add CERT test --- .github/workflows/main.yml | 1 + test/sqlancer/dbms/TestMySQLCERT.java | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 test/sqlancer/dbms/TestMySQLCERT.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 68ed03dc6..b1127faa7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -429,6 +429,7 @@ jobs: run: | MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP + MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLCERT oceanbase: name: DBMS Tests (OceanBase) diff --git a/test/sqlancer/dbms/TestMySQLCERT.java b/test/sqlancer/dbms/TestMySQLCERT.java new file mode 100644 index 000000000..a7704a545 --- /dev/null +++ b/test/sqlancer/dbms/TestMySQLCERT.java @@ -0,0 +1,22 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestMySQLCERT { + + @Test + public void testMySQL() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); + // Run with 0 queries as there are false positives for every mutation + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--max-expression-depth", "1", "--num-threads", "1", "--num-queries", "0", "mysql", "--oracle", + "CERT" })); + } + +} From 7fa58055f394f4511bb0e333686e03c460a7b781 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 14:13:30 +0800 Subject: [PATCH 586/856] Add common CERT oracle --- src/sqlancer/common/gen/CERTGenerator.java | 29 +++++ src/sqlancer/common/oracle/CERTOracle.java | 133 +++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100644 src/sqlancer/common/gen/CERTGenerator.java create mode 100644 src/sqlancer/common/oracle/CERTOracle.java diff --git a/src/sqlancer/common/gen/CERTGenerator.java b/src/sqlancer/common/gen/CERTGenerator.java new file mode 100644 index 000000000..b272ba4b2 --- /dev/null +++ b/src/sqlancer/common/gen/CERTGenerator.java @@ -0,0 +1,29 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public interface CERTGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + CERTGenerator setTablesAndColumns(AbstractTables tables); + + E generateBooleanExpression(); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + String generateExplainQuery(S select); + + boolean mutate(S select); +} diff --git a/src/sqlancer/common/oracle/CERTOracle.java b/src/sqlancer/common/oracle/CERTOracle.java new file mode 100644 index 000000000..48c174ef2 --- /dev/null +++ b/src/sqlancer/common/oracle/CERTOracle.java @@ -0,0 +1,133 @@ +package sqlancer.common.oracle; + +import java.io.IOException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.DBMSCommon; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +public class CERTOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private final CheckedFunction> rowCountParser; + private final CheckedFunction> queryPlanParser; + + private CERTGenerator gen; + private final ExpectedErrors errors; + + public CERTOracle(G state, CERTGenerator gen, ExpectedErrors expectedErrors, + CheckedFunction> rowCountParser, + CheckedFunction> queryPlanParser) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + this.rowCountParser = rowCountParser; + this.queryPlanParser = queryPlanParser; + } + + @Override + public void check() throws SQLException { + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + List fetchColumns = gen.generateFetchColumns(false); + + Z select = gen.generateSelect(); + select.setFetchColumns(fetchColumns); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateBooleanExpression()); + } + if (Randomly.getBoolean()) { + select.setGroupByClause(fetchColumns); + if (Randomly.getBoolean()) { + select.setHavingClause(gen.generateBooleanExpression()); + } + } + + List queryPlan1Sequences = new ArrayList<>(); + List queryPlan2Sequences = new ArrayList<>(); + + String queryString1 = gen.generateExplainQuery(select); + long rowCount1 = getRow(state, queryString1, queryPlan1Sequences); + + boolean increase = gen.mutate(select); + String queryString2 = gen.generateExplainQuery(select); + long rowCount2 = getRow(state, queryString2, queryPlan2Sequences); + + if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { + return; + } + + // Check the results + if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { + throw new AssertionError("Inconsistent result for query: " + queryString1 + "; --" + rowCount1 + "\n" + + queryString2 + "; --" + rowCount2); + } + } + + private Long getRow(SQLGlobalState globalState, String explainQuery, List queryPlanSequences) + throws AssertionError, SQLException { + Optional row = Optional.empty(); + + // Log the query + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(explainQuery); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + // Get the row count + SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + if (rs != null) { + while (rs.next()) { + Optional rowCount = rowCountParser.apply(rs); + if (row.isEmpty() && rowCount.isPresent()) { + row = rowCount; + } + + Optional queryPlanSequence = queryPlanParser.apply(rs); + queryPlanSequence.ifPresent(qps -> queryPlanSequences.add(qps)); + } + } + } catch (IgnoreMeException e) { + throw new IgnoreMeException(); + } catch (Exception e) { + throw new AssertionError(q.getQueryString(), e); + } + + return row.orElseThrow(IgnoreMeException::new); + } + + @FunctionalInterface + public interface CheckedFunction { + R apply(T t) throws SQLException; + } +} From 62a22d7bf80c41fcc9331f03f7c63916c1b7e2c9 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 14:14:01 +0800 Subject: [PATCH 587/856] [TiDB] Use common CERT oracle --- .../tidb/TiDBExpressionGenerator.java | 278 +++++++++++++----- src/sqlancer/tidb/TiDBOracleFactory.java | 17 +- src/sqlancer/tidb/oracle/TiDBCERTOracle.java | 267 ----------------- 3 files changed, 222 insertions(+), 340 deletions(-) delete mode 100644 src/sqlancer/tidb/oracle/TiDBCERTOracle.java diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index 0db285284..ff6a529d0 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -3,10 +3,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.CERTGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; @@ -31,6 +33,7 @@ import sqlancer.tidb.ast.TiDBFunctionCall; import sqlancer.tidb.ast.TiDBFunctionCall.TiDBFunction; import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBJoin.JoinType; import sqlancer.tidb.ast.TiDBOrderingTerm; import sqlancer.tidb.ast.TiDBRegexOperation; import sqlancer.tidb.ast.TiDBRegexOperation.TiDBRegexOperator; @@ -42,14 +45,8 @@ import sqlancer.tidb.ast.TiDBUnaryPrefixOperation.TiDBUnaryPrefixOperator; public class TiDBExpressionGenerator extends UntypedExpressionGenerator - implements TLPWhereGenerator { - - private final TiDBGlobalState globalState; - private List tables; - - public TiDBExpressionGenerator(TiDBGlobalState globalState) { - this.globalState = globalState; - } + implements TLPWhereGenerator, + CERTGenerator { private enum Gen { UNARY_PREFIX, // @@ -60,68 +57,12 @@ private enum Gen { // BINARY_ARITHMETIC } - @Override - protected TiDBExpression generateExpression(int depth) { - if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { - return generateLeafNode(); - } - if (allowAggregates && Randomly.getBoolean()) { - allowAggregates = false; - TiDBAggregateFunction func = TiDBAggregateFunction.getRandom(); - List args = generateExpressions(func.getNrArgs()); - return new TiDBAggregate(args, func); - } - switch (Randomly.fromOptions(Gen.values())) { - case DEFAULT: - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - TiDBColumn column = Randomly.fromList(columns); - if (column.hasDefault()) { - return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(new TiDBColumnReference(column))); - } - throw new IgnoreMeException(); - case UNARY_POSTFIX: - return new TiDBUnaryPostfixOperation(generateExpression(depth + 1), TiDBUnaryPostfixOperator.getRandom()); - case UNARY_PREFIX: - TiDBUnaryPrefixOperator rand = TiDBUnaryPrefixOperator.getRandom(); - return new TiDBUnaryPrefixOperation(generateExpression(depth + 1), rand); - case COLUMN: - return generateColumn(); - case CONSTANT: - return generateConstant(); - case COMPARISON: - return new TiDBBinaryComparisonOperation(generateExpression(depth + 1), generateExpression(depth + 1), - TiDBComparisonOperator.getRandom()); - case REGEX: - return new TiDBRegexOperation(generateExpression(depth + 1), generateExpression(depth + 1), - TiDBRegexOperator.getRandom()); - case FUNCTION: - TiDBFunction func = TiDBFunction.getRandom(); - return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); - case BINARY_BIT: - return new TiDBBinaryBitOperation(generateExpression(depth + 1), generateExpression(depth + 1), - TiDBBinaryBitOperator.getRandom()); - case BINARY_LOGICAL: - return new TiDBBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), - TiDBBinaryLogicalOperator.getRandom()); - case CAST: - return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions("BINARY", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 - "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 - "DECIMAL", "SIGNED", "UNSIGNED" /* https://github.com/pingcap/tidb/issues/16028 */)); - case CASE: - int nr = Randomly.fromOptions(1, 2); - return new TiDBCase(generateExpression(depth + 1), generateExpressions(nr, depth + 1), - generateExpressions(nr, depth + 1), generateExpression(depth + 1)); - default: - throw new AssertionError(); - } - } + private final TiDBGlobalState globalState; - @Override - protected TiDBExpression generateColumn() { - TiDBColumn column = Randomly.fromList(columns); - return new TiDBColumnReference(column); + private List tables; + + public TiDBExpressionGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; } @Override @@ -199,8 +140,7 @@ public TiDBExpression generateConstant(TiDBDataType type) { } @Override - public TLPWhereGenerator setTablesAndColumns( - AbstractTables tables) { + public TiDBExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -240,4 +180,200 @@ public List generateFetchColumns(boolean shouldCreateDummy) { return Randomly.nonEmptySubset(this.columns).stream().map(c -> new TiDBColumnReference(c)) .collect(Collectors.toList()); } + + @Override + protected TiDBExpression generateExpression(int depth) { + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBoolean()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBoolean()) { + allowAggregates = false; + TiDBAggregateFunction func = TiDBAggregateFunction.getRandom(); + List args = generateExpressions(func.getNrArgs()); + return new TiDBAggregate(args, func); + } + switch (Randomly.fromOptions(Gen.values())) { + case DEFAULT: + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + TiDBColumn column = Randomly.fromList(columns); + if (column.hasDefault()) { + return new TiDBFunctionCall(TiDBFunction.DEFAULT, Arrays.asList(new TiDBColumnReference(column))); + } + throw new IgnoreMeException(); + case UNARY_POSTFIX: + return new TiDBUnaryPostfixOperation(generateExpression(depth + 1), TiDBUnaryPostfixOperator.getRandom()); + case UNARY_PREFIX: + TiDBUnaryPrefixOperator rand = TiDBUnaryPrefixOperator.getRandom(); + return new TiDBUnaryPrefixOperation(generateExpression(depth + 1), rand); + case COLUMN: + return generateColumn(); + case CONSTANT: + return generateConstant(); + case COMPARISON: + return new TiDBBinaryComparisonOperation(generateExpression(depth + 1), generateExpression(depth + 1), + TiDBComparisonOperator.getRandom()); + case REGEX: + return new TiDBRegexOperation(generateExpression(depth + 1), generateExpression(depth + 1), + TiDBRegexOperator.getRandom()); + case FUNCTION: + TiDBFunction func = TiDBFunction.getRandom(); + return new TiDBFunctionCall(func, generateExpressions(func.getNrArgs(), depth)); + case BINARY_BIT: + return new TiDBBinaryBitOperation(generateExpression(depth + 1), generateExpression(depth + 1), + TiDBBinaryBitOperator.getRandom()); + case BINARY_LOGICAL: + return new TiDBBinaryLogicalOperation(generateExpression(depth + 1), generateExpression(depth + 1), + TiDBBinaryLogicalOperator.getRandom()); + case CAST: + return new TiDBCastOperation(generateExpression(depth + 1), Randomly.fromOptions("BINARY", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/52 + "CHAR", "DATE", "DATETIME", "TIME", // https://github.com/tidb-challenge-program/bug-hunting-issue/issues/13 + "DECIMAL", "SIGNED", "UNSIGNED" /* https://github.com/pingcap/tidb/issues/16028 */)); + case CASE: + int nr = Randomly.fromOptions(1, 2); + return new TiDBCase(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(); + } + } + + @Override + protected TiDBExpression generateColumn() { + TiDBColumn column = Randomly.fromList(columns); + return new TiDBColumnReference(column); + } + + @Override + public String generateExplainQuery(TiDBSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(TiDBSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateJoin); + mutators.add(this::mutateWhere); + if (!TiDBBugs.bug38319) { + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + } + mutators.add(this::mutateAnd); + if (!TiDBBugs.bug51525) { + mutators.add(this::mutateOr); + } + mutators.add(this::mutateLimit); + // mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateJoin(TiDBSelect select) { + if (select.getJoinList().isEmpty()) { + return false; + } + TiDBJoin join = (TiDBJoin) Randomly.fromList(select.getJoinList()); + if (join.getJoinType() == JoinType.NATURAL) { + return false; + } + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to + // other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((TiDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((TiDBTableReference) join.getRightTable()).getTable().getColumns()); + TiDBExpressionGenerator joinGen2 = new TiDBExpressionGenerator(globalState).setColumns(columns); + join.setOnCondition(joinGen2.generateExpression()); + } + + JoinType newJoinType = TiDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation + // between LEFT and RIGHT + // join + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); + } else { + newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); + } + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + if (newJoinType == JoinType.CROSS) { + join.setOnCondition(null); + } + return increase; + } + + boolean mutateWhere(TiDBSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression()); + } + return increase; + } + + boolean mutateHaving(TiDBSelect select) { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(TiDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + TiDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(TiDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + return false; + } else { + TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + TiDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + boolean mutateLimit(TiDBSelect select) { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(generateConstant(TiDBDataType.INT)); + } + return increase; + } + + private boolean mutateGroupBy(TiDBSelect select) { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + select.clearHavingClause(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } } diff --git a/src/sqlancer/tidb/TiDBOracleFactory.java b/src/sqlancer/tidb/TiDBOracleFactory.java index fac172adf..173ff7abd 100644 --- a/src/sqlancer/tidb/TiDBOracleFactory.java +++ b/src/sqlancer/tidb/TiDBOracleFactory.java @@ -3,13 +3,15 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import sqlancer.OracleFactory; +import sqlancer.common.oracle.CERTOracle; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.tidb.oracle.TiDBCERTOracle; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.tidb.oracle.TiDBDQPOracle; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; @@ -45,7 +47,18 @@ public TestOracle create(TiDBProvider.TiDBGlobalSt @Override public TestOracle create(TiDBProvider.TiDBGlobalState globalState) throws SQLException { - return new TiDBCERTOracle(globalState); + TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + String content = rs.getString(2); + return Optional.of((long) Double.parseDouble(content)); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String operation = rs.getString(1).split("_")[0]; // Extract operation names for query plans + return Optional.of(operation); + }; + + return new CERTOracle<>(globalState, gen, expectedErrors, rowCountParser, queryPlanParser); } @Override diff --git a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java b/src/sqlancer/tidb/oracle/TiDBCERTOracle.java deleted file mode 100644 index b55c1df1f..000000000 --- a/src/sqlancer/tidb/oracle/TiDBCERTOracle.java +++ /dev/null @@ -1,267 +0,0 @@ -package sqlancer.tidb.oracle; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLGlobalState; -import sqlancer.common.DBMSCommon; -import sqlancer.common.oracle.CERTOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.tidb.TiDBBugs; -import sqlancer.tidb.TiDBErrors; -import sqlancer.tidb.TiDBExpressionGenerator; -import sqlancer.tidb.TiDBProvider.TiDBGlobalState; -import sqlancer.tidb.TiDBSchema.TiDBColumn; -import sqlancer.tidb.TiDBSchema.TiDBDataType; -import sqlancer.tidb.TiDBSchema.TiDBTables; -import sqlancer.tidb.ast.TiDBBinaryLogicalOperation; -import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; -import sqlancer.tidb.ast.TiDBColumnReference; -import sqlancer.tidb.ast.TiDBExpression; -import sqlancer.tidb.ast.TiDBJoin; -import sqlancer.tidb.ast.TiDBJoin.JoinType; -import sqlancer.tidb.ast.TiDBSelect; -import sqlancer.tidb.ast.TiDBTableReference; -import sqlancer.tidb.visitor.TiDBVisitor; - -public class TiDBCERTOracle extends CERTOracleBase implements TestOracle { - private TiDBExpressionGenerator gen; - private TiDBSelect select; - - public TiDBCERTOracle(TiDBGlobalState globalState) { - super(globalState); - TiDBErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - queryPlan1Sequences = new ArrayList<>(); - queryPlan2Sequences = new ArrayList<>(); - - // Randomly generate a query - TiDBTables tables = state.getSchema().getRandomTableNonEmptyTables(); - gen = new TiDBExpressionGenerator(state).setColumns(tables.getColumns()); - select = new TiDBSelect(); - - List fetchColumns = new ArrayList<>(); - fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream().map(c -> new TiDBColumnReference(c)) - .collect(Collectors.toList())); - select.setFetchColumns(fetchColumns); - - List tableList = tables.getTables().stream().map(t -> new TiDBTableReference(t)) - .collect(Collectors.toList()); - List joins = TiDBJoin.getJoinsWithoutNature(tableList, state); - select.setJoinList(joins); - select.setFromList(tableList); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBys()); - } - if (Randomly.getBoolean()) { - select.setGroupByExpressions(select.getFetchColumns()); - if (Randomly.getBoolean()) { - select.setHavingClause(gen.generateHavingClause()); - } - } - if (Randomly.getBoolean()) { - select.setLimitClause(gen.generateExpression()); - } - if (Randomly.getBoolean()) { - select.setOffsetClause(gen.generateExpression()); - } - - // Get the result of the first query - String queryString1 = TiDBVisitor.asString(select); - double rowCount1 = getRow(state, queryString1, queryPlan1Sequences); - - List excludes = new ArrayList<>(); - excludes.add(Mutator.DISTINCT); - if (TiDBBugs.bug51525) { - excludes.add(Mutator.OR); - } - if (TiDBBugs.bug38319) { - excludes.add(Mutator.GROUPBY); - } - // Mutate the query - boolean increase = mutate(excludes.toArray(new Mutator[0])); - - // Get the result of the second query - String queryString2 = TiDBVisitor.asString(select); - double rowCount2 = getRow(state, queryString2, queryPlan2Sequences); - - // Check structural equivalence - if (!DBMSCommon.areQueryPlanSequencesSimilar(queryPlan1Sequences, queryPlan2Sequences)) { - return; - } - - /* - * https://github.com/pingcap/tidb/issues/38474 A minor issue in TiDB that some operations would round the - * result while others would not. A false alarm happens when both queries have the same number of estimated rows - * but the restrictued one rounds up to a bigger number. To avoid this issue until it is fixed, we make sure the - * gap between two estimated rows is at least 1. - */ - // Check the results - if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { - throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 - + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); - } - } - - @Override - protected boolean mutateJoin() { - if (select.getJoinList().isEmpty()) { - return false; - } - TiDBJoin join = (TiDBJoin) Randomly.fromList(select.getJoinList()); - - // CROSS does not need ON Condition, while other joins do - // To avoid Null pointer, generating a new new condition when mutating CROSS to - // other joins - if (join.getJoinType() == JoinType.CROSS) { - List columns = new ArrayList<>(); - columns.addAll(((TiDBTableReference) join.getLeftTable()).getTable().getColumns()); - columns.addAll(((TiDBTableReference) join.getRightTable()).getTable().getColumns()); - TiDBExpressionGenerator joinGen2 = new TiDBExpressionGenerator(state).setColumns(columns); - join.setOnCondition(joinGen2.generateExpression()); - } - - JoinType newJoinType = TiDBJoin.JoinType.INNER; - if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invarient relation - // between LEFT and RIGHT - // join - newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, JoinType.LEFT, JoinType.RIGHT); - } else { - newJoinType = JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); - } - assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT - boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); - join.setJoinType(newJoinType); - if (newJoinType == JoinType.CROSS) { - join.setOnCondition(null); - } - return increase; - } - - @Override - protected boolean mutateWhere() { - boolean increase = select.getWhereClause() != null; - if (increase) { - select.setWhereClause(null); - } else { - select.setWhereClause(gen.generateExpression()); - } - return increase; - } - - @Override - protected boolean mutateGroupBy() { - boolean increase = select.getGroupByExpressions().size() > 0; - if (increase) { - select.clearGroupByExpressions(); - select.clearHavingClause(); - } else { - select.setGroupByExpressions(select.getFetchColumns()); - } - return increase; - } - - @Override - protected boolean mutateHaving() { - if (select.getGroupByExpressions().size() == 0) { - select.setGroupByExpressions(select.getFetchColumns()); - select.setHavingClause(gen.generateExpression()); - return false; - } else { - if (select.getHavingClause() == null) { - select.setHavingClause(gen.generateExpression()); - return false; - } else { - select.setHavingClause(null); - return true; - } - } - } - - @Override - protected boolean mutateAnd() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression()); - } else { - TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), gen.generateExpression(), - TiDBBinaryLogicalOperator.AND); - select.setWhereClause(newWhere); - } - return false; - } - - @Override - protected boolean mutateOr() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression()); - return false; - } else { - TiDBExpression newWhere = new TiDBBinaryLogicalOperation(select.getWhereClause(), gen.generateExpression(), - TiDBBinaryLogicalOperator.OR); - select.setWhereClause(newWhere); - return true; - } - } - - @Override - protected boolean mutateLimit() { - boolean increase = select.getLimitClause() != null; - if (increase) { - select.setLimitClause(null); - } else { - select.setLimitClause(gen.generateConstant(TiDBDataType.INT)); - } - return increase; - } - - private double getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) - throws AssertionError, SQLException { - double row = -1; - String explainQuery = "EXPLAIN " + selectStr; - - // Log the query - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(explainQuery); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - // Get the row count - SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs != null) { - while (rs.next()) { - double estRows = Double.parseDouble(rs.getString(2)); - if (row == -1) { - row = estRows; - } - String operation = rs.getString(1).split("_")[0]; // Extract operation names for query plans - queryPlanSequences.add(operation); - return estRows; - } - } - } catch (Exception e) { - throw new AssertionError(q.getQueryString(), e); - } - if (row == -1) { - throw new IgnoreMeException(); - } - return row; - } -} From 856259c49ba2e88330dba5ad98d12ac91f549aea Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 15:22:45 +0800 Subject: [PATCH 588/856] [Postgres] Use common CERT oracle --- .../postgres/PostgresOracleFactory.java | 30 +- .../gen/PostgresExpressionGenerator.java | 140 ++++++++- .../postgres/oracle/PostgresCERTOracle.java | 272 ------------------ 3 files changed, 167 insertions(+), 275 deletions(-) delete mode 100644 src/sqlancer/postgres/oracle/PostgresCERTOracle.java diff --git a/src/sqlancer/postgres/PostgresOracleFactory.java b/src/sqlancer/postgres/PostgresOracleFactory.java index 45e2190fc..df2185776 100644 --- a/src/sqlancer/postgres/PostgresOracleFactory.java +++ b/src/sqlancer/postgres/PostgresOracleFactory.java @@ -3,16 +3,18 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import sqlancer.OracleFactory; +import sqlancer.common.oracle.CERTOracle; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.postgres.gen.PostgresCommon; import sqlancer.postgres.gen.PostgresExpressionGenerator; -import sqlancer.postgres.oracle.PostgresCERTOracle; import sqlancer.postgres.oracle.PostgresFuzzer; import sqlancer.postgres.oracle.PostgresPivotedQuerySynthesisOracle; import sqlancer.postgres.oracle.tlp.PostgresTLPAggregateOracle; @@ -72,7 +74,31 @@ public TestOracle create(PostgresGlobalState globalState) t CERT { @Override public TestOracle create(PostgresGlobalState globalState) throws SQLException { - return new PostgresCERTOracle(globalState); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + ExpectedErrors errors = ExpectedErrors.newErrors().with(PostgresCommon.getCommonExpressionErrors()) + .withRegex(PostgresCommon.getCommonExpressionRegexErrors()) + .with(PostgresCommon.getCommonFetchErrors()).with(PostgresCommon.getCommonInsertUpdateErrors()) + .with(PostgresCommon.getGroupingErrors()).with(PostgresCommon.getCommonInsertUpdateErrors()) + .with(PostgresCommon.getCommonRangeExpressionErrors()).build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + String content = rs.getString(1).trim(); + if (content.contains("rows=")) { + try { + int ind = content.indexOf("rows="); + long number = Long.parseLong(content.substring(ind + 5).split(" ")[0]); + return Optional.of(number); + } catch (Exception e) { + } + } + return Optional.empty(); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String content = rs.getString(1).trim(); + String[] planPart = content.split("-> "); + String plan = planPart[planPart.length - 1]; + return Optional.of(plan.split(" ")[0].trim()); + }; + return new CERTOracle<>(globalState, gen, errors, rowCountParser, queryPlanParser); } @Override diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 8c0c2352e..5f081ef2a 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -5,15 +5,18 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.CERTGenerator; import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.schema.AbstractTables; +import sqlancer.postgres.PostgresBugs; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresProvider; @@ -63,10 +66,12 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.ast.PostgresSimilarTo; +import sqlancer.postgres.ast.PostgresTableReference; public class PostgresExpressionGenerator implements ExpressionGenerator, NoRECGenerator, - TLPWhereGenerator { + TLPWhereGenerator, + CERTGenerator { private final int maxDepth; @@ -739,4 +744,137 @@ public String generateUnoptimizedQueryString(PostgresSelect select, PostgresExpr return "SELECT SUM(count) FROM (" + select.asString() + ") as res"; } + + @Override + public String generateExplainQuery(PostgresSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(PostgresSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateJoin); + mutators.add(this::mutateWhere); + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + if (!PostgresBugs.bug18643) { + mutators.add(this::mutateAnd); + mutators.add(this::mutateOr); + } + // mutators.add(this::mutateLimit); + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateJoin(PostgresSelect select) { + if (select.getJoinList().isEmpty()) { + return false; + } + PostgresJoin join = (PostgresJoin) Randomly.fromList(select.getJoinList()); + + // Exclude CROSS for on condition + if (join.getType() == PostgresJoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((PostgresTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((PostgresTableReference) join.getRightTable()).getTable().getColumns()); + PostgresExpressionGenerator joinGen2 = new PostgresExpressionGenerator(globalState).setColumns(columns); + join.setOnClause(joinGen2.generateExpression(0, PostgresDataType.BOOLEAN)); + } + + PostgresJoinType newJoinType = PostgresJoinType.INNER; + if (join.getType() == PostgresJoinType.LEFT || join.getType() == PostgresJoinType.RIGHT) { + newJoinType = PostgresJoinType.getRandomExcept(PostgresJoinType.LEFT, PostgresJoinType.RIGHT); + } else { + newJoinType = PostgresJoinType.getRandomExcept(join.getType()); + } + boolean increase = join.getType().ordinal() < newJoinType.ordinal(); + join.setType(newJoinType); + if (newJoinType == PostgresJoinType.CROSS) { + join.setOnClause(null); + } + return increase; + } + + boolean mutateDistinct(PostgresSelect select) { + PostgresSelect.SelectType selectType = select.getSelectOption(); + if (selectType != PostgresSelect.SelectType.ALL) { + select.setSelectType(PostgresSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(PostgresSelect.SelectType.DISTINCT); + return false; + } + } + + boolean mutateWhere(PostgresSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression(0, PostgresDataType.BOOLEAN)); + } + return increase; + } + + boolean mutateGroupBy(PostgresSelect select) { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + boolean mutateHaving(PostgresSelect select) { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(PostgresSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(0, PostgresDataType.BOOLEAN)); + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(PostgresSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(0, PostgresDataType.BOOLEAN)); + return false; + } else { + PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), + generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + boolean mutateLimit(PostgresSelect select) { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + Randomly r = new Randomly(); + select.setLimitClause(PostgresConstant.createIntConstant((int) Math.abs(r.getInteger()))); + } + return increase; + } } diff --git a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java b/src/sqlancer/postgres/oracle/PostgresCERTOracle.java deleted file mode 100644 index d22689816..000000000 --- a/src/sqlancer/postgres/oracle/PostgresCERTOracle.java +++ /dev/null @@ -1,272 +0,0 @@ -package sqlancer.postgres.oracle; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLGlobalState; -import sqlancer.common.DBMSCommon; -import sqlancer.common.oracle.CERTOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.PostgresBugs; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresSchema.PostgresColumn; -import sqlancer.postgres.PostgresSchema.PostgresDataType; -import sqlancer.postgres.PostgresSchema.PostgresTables; -import sqlancer.postgres.PostgresVisitor; -import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; -import sqlancer.postgres.ast.PostgresBinaryLogicalOperation.BinaryLogicalOperator; -import sqlancer.postgres.ast.PostgresColumnReference; -import sqlancer.postgres.ast.PostgresConstant; -import sqlancer.postgres.ast.PostgresExpression; -import sqlancer.postgres.ast.PostgresJoin; -import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; -import sqlancer.postgres.ast.PostgresSelect; -import sqlancer.postgres.ast.PostgresTableReference; -import sqlancer.postgres.gen.PostgresCommon; -import sqlancer.postgres.gen.PostgresExpressionGenerator; - -public class PostgresCERTOracle extends CERTOracleBase implements TestOracle { - private PostgresExpressionGenerator gen; - private PostgresSelect select; - - public PostgresCERTOracle(PostgresGlobalState globalState) { - super(globalState); - PostgresCommon.addCommonExpressionErrors(errors); - PostgresCommon.addCommonInsertUpdateErrors(errors); - PostgresCommon.addGroupingErrors(errors); - PostgresCommon.addCommonInsertUpdateErrors(errors); - PostgresCommon.addCommonRangeExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - queryPlan1Sequences = new ArrayList<>(); - queryPlan2Sequences = new ArrayList<>(); - - // Generate Random Query - PostgresTables tables = state.getSchema().getRandomTableNonEmptyTables(); - List tableList = tables.getTables().stream().map(t -> new PostgresTableReference(t)) - .collect(Collectors.toList()); - gen = new PostgresExpressionGenerator(state).setColumns(tables.getColumns()); - List fetchColumns = new ArrayList<>(); - fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() - .map(c -> new PostgresColumnReference(c)).collect(Collectors.toList())); - - select = new PostgresSelect(); - select.setFetchColumns(fetchColumns); - select.setFromList(tableList); - List joins = PostgresJoin.getJoins(tableList, state); - select.setJoinList(joins); - - select.setSelectType(Randomly.fromOptions(PostgresSelect.SelectType.values())); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - } - if (Randomly.getBoolean()) { - select.setGroupByExpressions(fetchColumns); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - } - } - - // First query row count - String queryString1 = PostgresVisitor.asString(select); - long rowCount1 = getRow(state, queryString1, queryPlan1Sequences); - - // JOIN and LIMIT mutations not added - boolean increase; - if (PostgresBugs.bug18643) { - increase = mutate(Mutator.LIMIT, Mutator.OR, Mutator.AND); - } else { - increase = mutate(Mutator.LIMIT); - } - - // Second Query row count - String queryString2 = PostgresVisitor.asString(select); - long rowCount2 = getRow(state, queryString2, queryPlan2Sequences); - - // Check query plan equivalence - if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { - return; - } - - // Check results - if (increase && rowCount1 > (rowCount2 + 1) || !increase && (rowCount1 + 1) < rowCount2) { - throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 - + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); - } - } - - @Override - protected boolean mutateJoin() { - if (select.getJoinList().isEmpty()) { - return false; - } - PostgresJoin join = (PostgresJoin) Randomly.fromList(select.getJoinList()); - - // Exclude CROSS for on condition - if (join.getType() == PostgresJoinType.CROSS) { - List columns = new ArrayList<>(); - columns.addAll(((PostgresTableReference) join.getLeftTable()).getTable().getColumns()); - columns.addAll(((PostgresTableReference) join.getRightTable()).getTable().getColumns()); - PostgresExpressionGenerator joinGen2 = new PostgresExpressionGenerator(state).setColumns(columns); - join.setOnClause(joinGen2.generateExpression(0, PostgresDataType.BOOLEAN)); - } - - PostgresJoinType newJoinType = PostgresJoinType.INNER; - if (join.getType() == PostgresJoinType.LEFT || join.getType() == PostgresJoinType.RIGHT) { - newJoinType = PostgresJoinType.getRandomExcept(PostgresJoinType.LEFT, PostgresJoinType.RIGHT); - } else { - newJoinType = PostgresJoinType.getRandomExcept(join.getType()); - } - boolean increase = join.getType().ordinal() < newJoinType.ordinal(); - join.setType(newJoinType); - if (newJoinType == PostgresJoinType.CROSS) { - join.setOnClause(null); - } - return increase; - } - - @Override - protected boolean mutateDistinct() { - PostgresSelect.SelectType selectType = select.getSelectOption(); - if (selectType != PostgresSelect.SelectType.ALL) { - select.setSelectType(PostgresSelect.SelectType.ALL); - return true; - } else { - select.setSelectType(PostgresSelect.SelectType.DISTINCT); - return false; - } - } - - @Override - protected boolean mutateWhere() { - boolean increase = select.getWhereClause() != null; - if (increase) { - select.setWhereClause(null); - } else { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - } - return increase; - } - - @Override - protected boolean mutateGroupBy() { - boolean increase = select.getGroupByExpressions().size() > 0; - if (increase) { - select.clearGroupByExpressions(); - } else { - select.setGroupByExpressions(select.getFetchColumns()); - } - return increase; - } - - @Override - protected boolean mutateHaving() { - if (select.getGroupByExpressions().size() == 0) { - select.setGroupByExpressions(select.getFetchColumns()); - select.setHavingClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - return false; - } else { - if (select.getHavingClause() == null) { - select.setHavingClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - return false; - } else { - select.setHavingClause(null); - return true; - } - } - } - - @Override - protected boolean mutateAnd() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - } else { - PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), - gen.generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.AND); - select.setWhereClause(newWhere); - } - return false; - } - - @Override - protected boolean mutateOr() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression(0, PostgresDataType.BOOLEAN)); - return false; - } else { - PostgresExpression newWhere = new PostgresBinaryLogicalOperation(select.getWhereClause(), - gen.generateExpression(0, PostgresDataType.BOOLEAN), BinaryLogicalOperator.OR); - select.setWhereClause(newWhere); - return true; - } - } - - @Override - protected boolean mutateLimit() { - boolean increase = select.getLimitClause() != null; - if (increase) { - select.setLimitClause(null); - } else { - Randomly r = new Randomly(); - select.setLimitClause(PostgresConstant.createIntConstant((int) Math.abs(r.getInteger()))); - } - return increase; - } - - private long getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) - throws AssertionError, SQLException { - long row = -1; - String explainQuery = "EXPLAIN " + selectStr; - - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(explainQuery); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - // Get row count - SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs != null) { - while (rs.next()) { - String content = rs.getString(1).trim(); - if (content.contains("rows=")) { - try { - int ind = content.indexOf("rows="); - long number = Long.parseLong(content.substring(ind + 5).split(" ")[0]); - if (row == -1) { - row = number; - - } - } catch (Exception e) { - } - } - // Proper Formatting TBD - String[] planPart = content.split("-> "); - String plan = planPart[planPart.length - 1]; - queryPlanSequences.add(plan.split(" ")[0].trim()); - } - - } - } catch (Exception e) { - throw new AssertionError(q.getQueryString(), e); - } - if (row == -1) { - throw new IgnoreMeException(); - } - return row; - - } -} From 2d54f09b1a3b375fded60d5e9b28ddf6a5f985b0 Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Tue, 8 Oct 2024 15:36:56 +0800 Subject: [PATCH 589/856] [MySQL] Use common CERT oracle --- src/sqlancer/mysql/MySQLOracleFactory.java | 19 +- .../mysql/gen/MySQLExpressionGenerator.java | 97 +++++++- .../mysql/oracle/MySQLCERTOracle.java | 217 ------------------ .../postgres/PostgresOracleFactory.java | 2 +- 4 files changed, 112 insertions(+), 223 deletions(-) delete mode 100644 src/sqlancer/mysql/oracle/MySQLCERTOracle.java diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index 206179101..a0170d685 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -1,13 +1,15 @@ package sqlancer.mysql; import java.sql.SQLException; +import java.util.Optional; import sqlancer.OracleFactory; +import sqlancer.common.oracle.CERTOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.mysql.gen.MySQLExpressionGenerator; -import sqlancer.mysql.oracle.MySQLCERTOracle; import sqlancer.mysql.oracle.MySQLDQPOracle; import sqlancer.mysql.oracle.MySQLFuzzer; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; @@ -40,7 +42,20 @@ public boolean requiresAllTablesToContainRows() { CERT { @Override public TestOracle create(MySQLGlobalState globalState) throws SQLException { - return new MySQLCERTOracle(globalState); + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + int rowCount = rs.getInt(10); + return Optional.of((long) rowCount); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String operation = rs.getString(2); + return Optional.of(operation); + }; + + return new CERTOracle<>(globalState, gen, expectedErrors, rowCountParser, queryPlanParser); + } @Override diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index bddf5946f..0958cca8a 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -2,10 +2,12 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.common.gen.CERTGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; @@ -41,7 +43,8 @@ import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; public class MySQLExpressionGenerator extends UntypedExpressionGenerator - implements TLPWhereGenerator { + implements TLPWhereGenerator, + CERTGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; @@ -208,8 +211,7 @@ public List generateOrderBys() { } @Override - public TLPWhereGenerator setTablesAndColumns( - AbstractTables tables) { + public MySQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); this.tables = tables.getTables(); @@ -240,4 +242,93 @@ public List getTableRefs() { public List generateFetchColumns(boolean shouldCreateDummy) { return columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); } + + @Override + public String generateExplainQuery(MySQLSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(MySQLSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateWhere); + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + mutators.add(this::mutateAnd); + mutators.add(this::mutateOr); + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateDistinct(MySQLSelect select) { + MySQLSelect.SelectType selectType = select.getFromOptions(); + if (selectType != MySQLSelect.SelectType.ALL) { + select.setSelectType(MySQLSelect.SelectType.ALL); + return true; + } else { + select.setSelectType(MySQLSelect.SelectType.DISTINCT); + return false; + } + } + + boolean mutateWhere(MySQLSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression()); + } + return increase; + } + + boolean mutateGroupBy(MySQLSelect select) { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + boolean mutateHaving(MySQLSelect select) { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression()); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression()); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(MySQLSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + MySQLBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(MySQLSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression()); + return false; + } else { + MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), generateExpression(), + MySQLBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } } diff --git a/src/sqlancer/mysql/oracle/MySQLCERTOracle.java b/src/sqlancer/mysql/oracle/MySQLCERTOracle.java deleted file mode 100644 index 2cdf3c7cf..000000000 --- a/src/sqlancer/mysql/oracle/MySQLCERTOracle.java +++ /dev/null @@ -1,217 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.io.IOException; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.SQLGlobalState; -import sqlancer.common.DBMSCommon; -import sqlancer.common.oracle.CERTOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.query.SQLancerResultSet; -import sqlancer.mysql.MySQLErrors; -import sqlancer.mysql.MySQLGlobalState; -import sqlancer.mysql.MySQLSchema.MySQLTables; -import sqlancer.mysql.MySQLVisitor; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; -import sqlancer.mysql.ast.MySQLColumnReference; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLSelect; -import sqlancer.mysql.ast.MySQLTableReference; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -public class MySQLCERTOracle extends CERTOracleBase implements TestOracle { - private MySQLExpressionGenerator gen; - private MySQLSelect select; - - public MySQLCERTOracle(MySQLGlobalState globalState) { - super(globalState); - MySQLErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws SQLException { - queryPlan1Sequences = new ArrayList<>(); - queryPlan2Sequences = new ArrayList<>(); - - // Randomly generate a query - MySQLTables tables = state.getSchema().getRandomTableNonEmptyTables(); - gen = new MySQLExpressionGenerator(state).setColumns(tables.getColumns()); - List fetchColumns = new ArrayList<>(); - fetchColumns.addAll(Randomly.nonEmptySubset(tables.getColumns()).stream() - .map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList())); - List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) - .collect(Collectors.toList()); - - select = new MySQLSelect(); - select.setFetchColumns(fetchColumns); - select.setFromList(tableList); - - select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression()); - } - if (Randomly.getBoolean()) { - select.setGroupByExpressions(fetchColumns); - if (Randomly.getBoolean()) { - select.setHavingClause(gen.generateExpression()); - } - } - - // Set the join. Todo: to make it random - // List joinExpressions = getJoins(tableList, state); - // select.setJoinList(joinExpressions); - - // Get the result of the first query - String queryString1 = MySQLVisitor.asString(select); - int rowCount1 = getRow(state, queryString1, queryPlan1Sequences); - - boolean increase = mutate(Mutator.JOIN, Mutator.LIMIT); - - // Get the result of the second query - String queryString2 = MySQLVisitor.asString(select); - int rowCount2 = getRow(state, queryString2, queryPlan2Sequences); - - // Check structural equivalence - if (DBMSCommon.editDistance(queryPlan1Sequences, queryPlan2Sequences) > 1) { - return; - } - - // Check the results - if (increase && rowCount1 > rowCount2 || !increase && rowCount1 < rowCount2) { - throw new AssertionError("Inconsistent result for query: EXPLAIN " + queryString1 + "; --" + rowCount1 - + "\nEXPLAIN " + queryString2 + "; --" + rowCount2); - } - } - - @Override - protected boolean mutateDistinct() { - MySQLSelect.SelectType selectType = select.getFromOptions(); - if (selectType != MySQLSelect.SelectType.ALL) { - select.setSelectType(MySQLSelect.SelectType.ALL); - return true; - } else { - select.setSelectType(MySQLSelect.SelectType.DISTINCT); - return false; - } - } - - @Override - protected boolean mutateWhere() { - boolean increase = select.getWhereClause() != null; - if (increase) { - select.setWhereClause(null); - } else { - select.setWhereClause(gen.generateExpression()); - } - return increase; - } - - @Override - protected boolean mutateGroupBy() { - boolean increase = select.getGroupByExpressions().size() > 0; - if (increase) { - select.clearGroupByExpressions(); - } else { - select.setGroupByExpressions(select.getFetchColumns()); - } - return increase; - } - - @Override - protected boolean mutateHaving() { - if (select.getGroupByExpressions().size() == 0) { - select.setGroupByExpressions(select.getFetchColumns()); - select.setHavingClause(gen.generateExpression()); - return false; - } else { - if (select.getHavingClause() == null) { - select.setHavingClause(gen.generateExpression()); - return false; - } else { - select.setHavingClause(null); - return true; - } - } - } - - @Override - protected boolean mutateAnd() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression()); - } else { - MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), - gen.generateExpression(), MySQLBinaryLogicalOperator.AND); - select.setWhereClause(newWhere); - } - return false; - } - - @Override - protected boolean mutateOr() { - if (select.getWhereClause() == null) { - select.setWhereClause(gen.generateExpression()); - return false; - } else { - MySQLExpression newWhere = new MySQLBinaryLogicalOperation(select.getWhereClause(), - gen.generateExpression(), MySQLBinaryLogicalOperator.OR); - select.setWhereClause(newWhere); - return true; - } - } - - // The limit clause only accpets positive integers, which is not supported yet - // private boolean mutateLimit() { - // boolean increase = select.getLimitClause() != null; - // if (increase) { - // select.setLimitClause(null); - // } else { - // select.setLimitClause(gen.generateConstant()); - // } - // return increase; - // } - - private int getRow(SQLGlobalState globalState, String selectStr, List queryPlanSequences) - throws AssertionError, SQLException { - int row = -1; - String explainQuery = "EXPLAIN " + selectStr; - - // Log the query - if (globalState.getOptions().logEachSelect()) { - globalState.getLogger().writeCurrent(explainQuery); - try { - globalState.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - e.printStackTrace(); - } - } - - // Get the row count - SQLQueryAdapter q = new SQLQueryAdapter(explainQuery, errors); - try (SQLancerResultSet rs = q.executeAndGet(globalState)) { - if (rs != null) { - while (rs.next()) { - int estRows = rs.getInt(10); - if (row == -1) { - row = estRows; - } - String operation = rs.getString(2); - queryPlanSequences.add(operation); - } - } - } catch (Exception e) { - throw new AssertionError(q.getQueryString(), e); - } - if (row == -1) { - throw new IgnoreMeException(); - } - return row; - } - -} diff --git a/src/sqlancer/postgres/PostgresOracleFactory.java b/src/sqlancer/postgres/PostgresOracleFactory.java index df2185776..7c00021ed 100644 --- a/src/sqlancer/postgres/PostgresOracleFactory.java +++ b/src/sqlancer/postgres/PostgresOracleFactory.java @@ -82,7 +82,7 @@ public TestOracle create(PostgresGlobalState globalState) t .with(PostgresCommon.getCommonRangeExpressionErrors()).build(); CERTOracle.CheckedFunction> rowCountParser = (rs) -> { String content = rs.getString(1).trim(); - if (content.contains("rows=")) { + if (content.contains("Result") && content.contains("rows=")) { try { int ind = content.indexOf("rows="); long number = Long.parseLong(content.substring(ind + 5).split(" ")[0]); From 0d70f35b97e5b10e482437350c45999bc153fa7c Mon Sep 17 00:00:00 2001 From: malwaregarry Date: Wed, 9 Oct 2024 20:46:40 +0800 Subject: [PATCH 590/856] [CockroachDB] Use common CERT oracle --- .../cockroachdb/CockroachDBErrors.java | 1 + .../cockroachdb/CockroachDBOracleFactory.java | 33 +++- .../gen/CockroachDBExpressionGenerator.java | 142 +++++++++++++++++- test/sqlancer/dbms/TestCockroachDBCERT.java | 2 +- 4 files changed, 174 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBErrors.java b/src/sqlancer/cockroachdb/CockroachDBErrors.java index 7a06fe51e..134ad35aa 100644 --- a/src/sqlancer/cockroachdb/CockroachDBErrors.java +++ b/src/sqlancer/cockroachdb/CockroachDBErrors.java @@ -351,6 +351,7 @@ private static List getComputedColumnErrors() { // computed columns errors.add("computed column expressions cannot reference computed columns"); + errors.add("STORED COMPUTED COLUMN expression cannot reference computed columns"); return errors; } diff --git a/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java index bbf261f16..96fbc22ce 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java +++ b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java @@ -3,20 +3,23 @@ import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.util.Optional; +import sqlancer.IgnoreMeException; import sqlancer.OracleFactory; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; -import sqlancer.cockroachdb.oracle.CockroachDBCERTOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPGroupByOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPHavingOracle; +import sqlancer.common.oracle.CERTOracle; import sqlancer.common.oracle.CompositeTestOracle; import sqlancer.common.oracle.NoRECOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLancerResultSet; public enum CockroachDBOracleFactory implements OracleFactory { NOREC { @@ -97,7 +100,33 @@ public TestOracle create( @Override public TestOracle create( CockroachDBProvider.CockroachDBGlobalState globalState) throws SQLException { - return new CockroachDBCERTOracle(globalState); + CockroachDBExpressionGenerator gen = new CockroachDBExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(CockroachDBErrors.getExpressionErrors()) + .build(); + CERTOracle.CheckedFunction> rowCountParser = (rs) -> { + String content = rs.getString(1); + if (content.contains("count:")) { + try { + long number = Long.parseLong(content.split("count: ")[1].split(" ")[0].replace(",", "")); + return Optional.of(number); + } catch (Exception e) { // To avoid the situation that no number is found + } + } + return Optional.empty(); + }; + CERTOracle.CheckedFunction> queryPlanParser = (rs) -> { + String content = rs.getString(1); + if (content.contains("• ")) { + String operation = content.split("• ")[1].split(" ")[0]; + if (CockroachDBBugs.bug131875 && (operation.equals("distinct") || operation.equals("limit"))) { + throw new IgnoreMeException(); + } + return Optional.of(operation); + } + return Optional.empty(); + }; + + return new CERTOracle<>(globalState, gen, expectedErrors, rowCountParser, queryPlanParser); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index 189be028d..52d8f027d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -2,9 +2,11 @@ import java.util.ArrayList; import java.util.List; +import java.util.function.Function; import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBBugs; import sqlancer.cockroachdb.CockroachDBCommon; import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; @@ -32,6 +34,7 @@ import sqlancer.cockroachdb.ast.CockroachDBFunction; import sqlancer.cockroachdb.ast.CockroachDBInOperation; import sqlancer.cockroachdb.ast.CockroachDBJoin; +import sqlancer.cockroachdb.ast.CockroachDBJoin.JoinType; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonOperator; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison.MultiValuedComparisonType; @@ -44,6 +47,7 @@ import sqlancer.cockroachdb.ast.CockroachDBTypeAnnotation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation; import sqlancer.cockroachdb.ast.CockroachDBUnaryPostfixOperation.CockroachDBUnaryPostfixOperator; +import sqlancer.common.gen.CERTGenerator; import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.TypedExpressionGenerator; @@ -52,7 +56,8 @@ public class CockroachDBExpressionGenerator extends TypedExpressionGenerator implements NoRECGenerator, - TLPWhereGenerator { + TLPWhereGenerator, + CERTGenerator { private List tables; private final CockroachDBGlobalState globalState; @@ -458,4 +463,139 @@ public List generateFetchColumns(boolean shouldCreateDumm return Randomly.nonEmptySubset(columns).stream().map(c -> new CockroachDBColumnReference(c)) .collect(Collectors.toList()); } + + @Override + public String generateExplainQuery(CockroachDBSelect select) { + return "EXPLAIN " + select.asString(); + } + + @Override + public boolean mutate(CockroachDBSelect select) { + List> mutators = new ArrayList<>(); + + if (!CockroachDBBugs.bug131647) { + mutators.add(this::mutateJoin); + } + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + mutators.add(this::mutateAnd); + if (!CockroachDBBugs.bug131640) { + mutators.add(this::mutateWhere); + mutators.add(this::mutateOr); + } + // mutators.add(this::mutateLimit); + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + boolean mutateJoin(CockroachDBSelect select) { + if (select.getJoinList().isEmpty()) { + return false; + } + + CockroachDBJoin join = (CockroachDBJoin) Randomly.fromList(select.getJoinList()); + + // CROSS does not need ON Condition, while other joins do + // To avoid Null pointer, generating a new new condition when mutating CROSS to other joins + if (join.getJoinType() == JoinType.CROSS) { + List columns = new ArrayList<>(); + columns.addAll(((CockroachDBTableReference) join.getLeftTable()).getTable().getColumns()); + columns.addAll(((CockroachDBTableReference) join.getRightTable()).getTable().getColumns()); + CockroachDBExpressionGenerator joinGen2 = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + join.setOnClause(joinGen2.generateExpression(CockroachDBDataType.BOOL.get())); + } + + JoinType newJoinType = CockroachDBJoin.JoinType.INNER; + if (join.getJoinType() == JoinType.LEFT || join.getJoinType() == JoinType.RIGHT) { // No invariant relation + // between LEFT and RIGHT + // join + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS, JoinType.LEFT, + JoinType.RIGHT); + } else if (join.getJoinType() == JoinType.FULL) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, JoinType.CROSS); + } else if (join.getJoinType() != JoinType.CROSS) { + newJoinType = CockroachDBJoin.JoinType.getRandomExcept(JoinType.NATURAL, join.getJoinType()); + } + assert newJoinType != JoinType.NATURAL; // Natural Join is not supported for CERT + boolean increase = join.getJoinType().ordinal() < newJoinType.ordinal(); + join.setJoinType(newJoinType); + return increase; + } + + boolean mutateDistinct(CockroachDBSelect select) { + boolean increase = select.isDistinct(); + select.setDistinct(!select.isDistinct()); + return increase; + } + + boolean mutateWhere(CockroachDBSelect select) { + boolean increase = select.getWhereClause() != null; + if (increase) { + select.setWhereClause(null); + } else { + select.setWhereClause(generateExpression(CockroachDBDataType.BOOL.get())); + } + return increase; + } + + boolean mutateGroupBy(CockroachDBSelect select) { + boolean increase = select.getGroupByExpressions().size() > 0; + if (increase) { + select.clearGroupByExpressions(); + } else { + select.setGroupByExpressions(select.getFetchColumns()); + } + return increase; + } + + boolean mutateHaving(CockroachDBSelect select) { + if (select.getGroupByExpressions().size() == 0) { + select.setGroupByExpressions(select.getFetchColumns()); + select.setHavingClause(generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + if (select.getHavingClause() == null) { + select.setHavingClause(generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + select.setHavingClause(null); + return true; + } + } + } + + boolean mutateAnd(CockroachDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(CockroachDBDataType.BOOL.get())); + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.AND); + select.setWhereClause(newWhere); + } + return false; + } + + boolean mutateOr(CockroachDBSelect select) { + if (select.getWhereClause() == null) { + select.setWhereClause(generateExpression(CockroachDBDataType.BOOL.get())); + return false; + } else { + CockroachDBExpression newWhere = new CockroachDBBinaryLogicalOperation(select.getWhereClause(), + generateExpression(CockroachDBDataType.BOOL.get()), CockroachDBBinaryLogicalOperator.OR); + select.setWhereClause(newWhere); + return true; + } + } + + boolean mutateLimit(CockroachDBSelect select) { + boolean increase = select.getLimitClause() != null; + if (increase) { + select.setLimitClause(null); + } else { + select.setLimitClause(generateConstant(CockroachDBDataType.INT.get())); + } + return increase; + } } diff --git a/test/sqlancer/dbms/TestCockroachDBCERT.java b/test/sqlancer/dbms/TestCockroachDBCERT.java index 7e60818c1..4d7140e40 100644 --- a/test/sqlancer/dbms/TestCockroachDBCERT.java +++ b/test/sqlancer/dbms/TestCockroachDBCERT.java @@ -13,7 +13,7 @@ public class TestCockroachDBCERT { public void testCockroachDBCERT() { assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.COCKROACHDB_ENV)); assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "CERT" })); + "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, "cockroachdb", "--oracle", "CERT" })); } } From 2dd2e1a9396e697877882f252b1a3a447c04614a Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Sun, 8 Dec 2024 20:11:55 +1030 Subject: [PATCH 591/856] Ignore database* to gitignore - Skip make package outputs --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index a3ecf6076..ff30cb6de 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,7 @@ SQLancer.iml dependency-reduced-pom.xml database0.db databaseconnectiontest.db +database*.log +database*.properties +database*.script +databases/ \ No newline at end of file From e501451c5ef80c303ff4ee3911bfec4102079038 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Jan 2025 14:59:24 +0800 Subject: [PATCH 592/856] Remove the Twitter/X link since the account is no longer actively being maintained --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index b0fe4769b..a18c855c8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,4 @@ [![Build Status](https://github.com/sqlancer/sqlancer/workflows/ci/badge.svg)](https://github.com/sqlancer/sqlancer/actions) -[![Twitter](https://img.shields.io/twitter/follow/sqlancer_dbms?style=social)](https://twitter.com/sqlancer_dbms) -# SQLancer ![SQLancer](media/logo/png/sqlancer_logo_logo_pos_500.png) From 63e69e4deda890417572d758d37920abe9e4e642 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Jan 2025 16:51:57 +0800 Subject: [PATCH 593/856] Give a better overview of SQLancer --- README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a18c855c8..596d6545f 100644 --- a/README.md +++ b/README.md @@ -3,12 +3,11 @@ ![SQLancer](media/logo/png/sqlancer_logo_logo_pos_500.png) -SQLancer (Synthesized Query Lancer) is a tool to automatically test Database Management Systems (DBMS) in order to find logic bugs in their implementation. We refer to logic bugs as those bugs that cause the DBMS to fetch an incorrect result set (e.g., by omitting a record). +SQLancer is a tool to automatically test Database Management Systems (DBMSs) in order to find bugs in their implementation. That is, it finds bugs in the code of the DBMS implementation, rather than in queries written by the user. SQLancer has found hundreds of bugs in mature and widely-known DBMSs. -SQLancer operates in the following two phases: - -1. Database generation: The goal of this phase is to create a populated database, and stress the DBMS to increase the probability of causing an inconsistent database state that could be detected subsequently. First, random tables are created. Then, randomly SQL statements are chosen to generate, modify, and delete data. Also other statements, such as those to create indexes as well as views and to set DBMS-specific options are sent to the DBMS. -2. Testing: The goal of this phase is to detect the logic bugs based on the generated database. See Testing Approaches below. **News: we support Differential Query Plans (DQP) oracle now. See Testing Approaches below.** +SQLancer tackles two essential challenges when automatically testing the DBMSs: +1. **Test input generation**: SQLancer implements approaches for automatically generating SQL statements. It contains various hand-written SQL generators that operate in multiple phases. First, a database schema is created, which refers to a set of tables and their columns. Then, data is inserted into these tables, along with creating various other kinds of database states such as indexes, views, or database-specific options. Finally, queries are generated, which can be validated using one of multiple result validators (also called *test oracles*) that SQLancer provides. Besides the standard approach of creating the statements in an unguided way, SQLancer also supports a test input-generation approach that is feedback-guided and aims to exercise as many unique query plans as possible based on the intuition that doing so would exercise many interesting behaviors in the database system [[ICSE '23]](https://arxiv.org/pdf/2312.17510). +2. **Test oracles**: A key innovation in SQLancer is that it provides ways to find deep kinds of bugs in DBMSs. As a main focus, it can find logic bugs, which are bugs that cause the DBMS to fetch an incorrect result set (e.g., by omitting a record). We have proposed multiple complementary test oracles such as *Ternary Logic Partitioning (TLP)* [[OOPSLA '20]](https://dl.acm.org/doi/pdf/10.1145/3428279), *Non-optimizing Reference Engine Construction (NoREC)* [[ESEC/FSE 2020]](https://arxiv.org/abs/2007.08292), *Pivoted Query Synthesis (PQS)* [[OSDI '20]](https://www.usenix.org/system/files/osdi20-rigger.pdf), *Differential Query Plans (DQP)* [[SIGMOD '24]](https://dl.acm.org/doi/pdf/10.1145/3654991), and *Constant Optimization Driven Database System Testing (CODDTest)* [SIGMOD '25]. It can also find specific categories of performance issues, which refer to cases where a DBMS could reasonably be expected to produce its result more efficiently using a technique called *Cardinality Estimation Restriction Testing (CERT)* [[ICSE '24]](https://arxiv.org/pdf/2306.00355). SQLancer can detect unexpected internal errors (e.g., an error that the database is corrupted) by declaring all potential errors that might be returned by a DBMS for a given query. Finally, SQLancer can find crash bugs, which are bugs that cause the DBMS process to terminate. For this, it uses an implicit test oracle. # Getting Started From 4a9f1d7b2dd62c37bc9b6e80f0d1c5f705654e66 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Jan 2025 17:14:47 +0800 Subject: [PATCH 594/856] Update the getting started guide --- README.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 596d6545f..1eb7bac1e 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,11 @@ SQLancer tackles two essential challenges when automatically testing the DBMSs: # Getting Started -Requirements: +Minimum Requirements: * Java 11 or above -* [Maven](https://maven.apache.org/) (`sudo apt install maven` on Ubuntu) -* The DBMS that you want to test (embedded DBMSs such as DuckDB, H2, and SQLite do not require a setup) +* [Maven](https://maven.apache.org/) -The following commands clone SQLancer, create a JAR, and start SQLancer to test SQLite using Non-optimizing Reference Engine Construction (NoREC): +The following commands clone SQLancer, create a JAR, and start SQLancer to test SQLite using [Non-optimizing Reference Engine Construction (NoREC)](https://arxiv.org/abs/2007.08292): ``` git clone https://github.com/sqlancer/sqlancer @@ -26,9 +25,12 @@ cd target java -jar sqlancer-*.jar --num-threads 4 sqlite3 --oracle NoREC ``` -If the execution prints progress information every five seconds, then the tool works as expected. Note that SQLancer might find bugs in SQLite. Before reporting these, be sure to check that they can still be reproduced when using the latest development version. The shortcut CTRL+C can be used to terminate SQLancer manually. If SQLancer does not find any bugs, it executes infinitely. The option `--num-tries` can be used to control after how many bugs SQLancer terminates. Alternatively, the option `--timeout-seconds` can be used to specify the maximum duration that SQLancer is allowed to run. +**Running and terminating.** If the execution prints progress information every five seconds, then the tool works as expected. The shortcut CTRL+C can be used to terminate SQLancer manually. If SQLancer does not find any bugs, it executes infinitely. The option `--num-tries` can be used to control after how many bugs SQLancer terminates. Alternatively, the option `--timeout-seconds` can be used to specify the maximum duration that SQLancer is allowed to run. + +**Parameters.** If you launch SQLancer without parameters, available options and commands are displayed. Note that general options that are supported by all DBMS-testing implementations (e.g., `--num-threads`) need to precede the name of the DBMS to be tested (e.g., `sqlite3`). Options that are supported only for specific DBMS (e.g., `--test-rtree` for SQLite3), or options for which each testing implementation provides different values (e.g. `--oracle NoREC`) need to go after the DBMS name. + +**DBMSs.** To run SQLancer on SQLite, it was not necessary to install and set up a DBMS. The reason for this is that embedded DBMSs run in the same process as the application and thus require no separate installation or setup. Embedded DBMSs supported by SQLancer include DuckDB, H2, and SQLite. Their binaries are included as [JAR dependencies](https://github.com/sqlancer/sqlancer/blob/main/pom.xml). Note that any crashes in these systems will also cause a crash in the JVM on which SQLancer runs. -If you launch SQLancer without parameters, available options and commands are displayed. Note that general options that are supported by all DBMS-testing implementations (e.g., `--num-threads`) need to precede the name of DBMS to be tested (e.g., `sqlite3`). Options that are supported only for specific DBMS (e.g., `--test-rtree` for SQLite3), or options for which each testing implementation provides different values (e.g. `--oracle NoREC`) need to go after the DBMS name. # Testing Approaches @@ -98,6 +100,8 @@ After finding a bug, it is useful to produce a minimal test case before reportin ## Found Bugs +For most DBMSs, SQLancer supports only a previous *release* version. Thus, potential bugs that SQLancer finds could be already fixed in the latest *development* version of the DBMS. If you are not a developer of the DBMS that you are testing, we would like to encourage you to validate that the bug can still be reproduced before reporting it. + We would appreciate it if you mention SQLancer when you report bugs found by it. We would also be excited to know if you are using SQLancer to find bugs, or if you have extended it to test another DBMS (also if you do not plan to contribute it to this project). SQLancer has found over 400 bugs in widely-used DBMS, which are listed [here](https://www.manuelrigger.at/dbms-bugs/). From 002b9a4885d0afb7aa2ae8866a870d1c2b86c69a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Jan 2025 17:39:38 +0800 Subject: [PATCH 595/856] Restructure and update the section on how to use SQLancer --- README.md | 59 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 1eb7bac1e..27f9579e1 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,41 @@ java -jar sqlancer-*.jar --num-threads 4 sqlite3 --oracle NoREC **DBMSs.** To run SQLancer on SQLite, it was not necessary to install and set up a DBMS. The reason for this is that embedded DBMSs run in the same process as the application and thus require no separate installation or setup. Embedded DBMSs supported by SQLancer include DuckDB, H2, and SQLite. Their binaries are included as [JAR dependencies](https://github.com/sqlancer/sqlancer/blob/main/pom.xml). Note that any crashes in these systems will also cause a crash in the JVM on which SQLancer runs. +# Using SQLancer + +**Logs.** SQLancer stores logs in the `target/logs` subdirectory. By default, the option `--log-each-select` is enabled, which results in every SQL statement that is sent to the DBMS being logged. The corresponding file names are postfixed with `-cur.log`. In addition, if SQLancer detects a logic bug, it creates a file with the extension `.log`, in which the statements to reproduce the bug are logged, including only the last query that was executed along with the other statements to set up the database state. + +**Reducing bugs.** After finding a bug-inducing test input, the input typically needs to be reduced to be further analyzed, as it might contain many SQL statements that are redundant to reproduce the bug. One option is to do this manually, by removing a statement or feature at a time, replaying the bug-inducing statements, and applying the test oracle (e.g., for test oracles like TLP or NoREC, this would require checking that both queries still produce a different result). This process can be automated using a so-called [delta-debugging approach](https://www.debuggingbook.org/html/DeltaDebugger.html). SQLancer includes an experimental implementation of a delta debugging approach, which can be enabled using `--use-reducer`. In the past, we have successfully used [C-Reduce](https://embed.cs.utah.edu/creduce/), which requires specifying the test oracle in a script that can be executed by C-Reduce. + +**Testing the latest DBMS version.** For most DBMSs, SQLancer supports only a previous *release* version. Thus, potential bugs that SQLancer finds could be already fixed in the latest *development* version of the DBMS. If you are not a developer of the DBMS that you are testing, we would like to encourage you to validate that the bug can still be reproduced before reporting it. We would appreciate it if you could mention SQLancer when you report bugs found by it. We would also be excited to hear about your experience using SQLancer or related use cases or extensions. + +**Options.** SQLancer provides many options that you can use to customize its behavior. Executing `java -jar sqlancer-*.jar --help` will list them and should print output such as the following: +``` +Usage: SQLancer [options] [command] [command options] + Options: + --ast-reducer-max-steps + EXPERIMENTAL Maximum steps the AST-based reducer will do + Default: -1 + --ast-reducer-max-time + EXPERIMENTAL Maximum time duration (secs) the statement reducer will do + Default: -1 + --canonicalize-sql-strings + Should canonicalize query string (add ';' at the end + Default: true + --constant-cache-size + Specifies the size of the constant cache. This option only takes effect + when constant caching is enabled + Default: 100 +... +``` + +**Which SQLancer version to use.** The recommended way to use SQLancer is to use its latest source version on GitHub. Infrequent and irregular official releases are also available on the following platforms: +* [GitHub](https://github.com/sqlancer/sqlancer/releases) +* [Maven Central](https://search.maven.org/artifact/com.sqlancer/sqlancer) +* [DockerHub](https://hub.docker.com/r/mrigger/sqlancer) + +**Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. + # Testing Approaches | Approach | Description | @@ -88,23 +123,6 @@ Some DBMS were once supported but subsequently removed. | StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. | -# Using SQLancer - -## Logs - -SQLancer stores logs in the `target/logs` subdirectory. By default, the option `--log-each-select` is enabled, which results in every SQL statement that is sent to the DBMS being logged. The corresponding file names are postfixed with `-cur.log`. In addition, if SQLancer detects a logic bug, it creates a file with the extension `.log`, in which the statements to reproduce the bug are logged. - -## Reducing a Bug - -After finding a bug, it is useful to produce a minimal test case before reporting the bug, to save the DBMS developers' time and effort. For many test cases, [C-Reduce](https://embed.cs.utah.edu/creduce/) does a great job. - -## Found Bugs - -For most DBMSs, SQLancer supports only a previous *release* version. Thus, potential bugs that SQLancer finds could be already fixed in the latest *development* version of the DBMS. If you are not a developer of the DBMS that you are testing, we would like to encourage you to validate that the bug can still be reproduced before reporting it. - -We would appreciate it if you mention SQLancer when you report bugs found by it. We would also be excited to know if you are using SQLancer to find bugs, or if you have extended it to test another DBMS (also if you do not plan to contribute it to this project). SQLancer has found over 400 bugs in widely-used DBMS, which are listed [here](https://www.manuelrigger.at/dbms-bugs/). - - # Community We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. SQLancer's official Twitter handle is [@sqlancer_dbms](https://twitter.com/sqlancer_dbms). @@ -126,13 +144,6 @@ For some DBMSs, SQLancer expects that a database "test" exists, which it then us * [Contributing to SQLancer](CONTRIBUTING.md) * [Papers and .bib entries](docs/PAPERS.md) -# Releases - -Official release are available on: -* [GitHub](https://github.com/sqlancer/sqlancer/releases) -* [Maven Central](https://search.maven.org/artifact/com.sqlancer/sqlancer) -* [DockerHub](https://hub.docker.com/r/mrigger/sqlancer) - # Additional Resources * A talk on Ternary Logic Partitioning (TLP) and SQLancer is available on [YouTube](https://www.youtube.com/watch?v=Np46NQ6lqP8). From bfad1c33b80ef06b0142c5f0659000b027a7feaa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 23:47:55 +0800 Subject: [PATCH 596/856] Bump org.xerial:sqlite-jdbc from 3.46.1.0 to 3.47.2.0 (#1052) Bumps [org.xerial:sqlite-jdbc](https://github.com/xerial/sqlite-jdbc) from 3.46.1.0 to 3.47.2.0. - [Release notes](https://github.com/xerial/sqlite-jdbc/releases) - [Changelog](https://github.com/xerial/sqlite-jdbc/blob/master/CHANGELOG) - [Commits](https://github.com/xerial/sqlite-jdbc/compare/3.46.1.0...3.47.2.0) --- updated-dependencies: - dependency-name: org.xerial:sqlite-jdbc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 119ef6434..1bcf92957 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ org.xerial sqlite-jdbc - 3.46.1.0 + 3.47.2.0 mysql From 2814b206e392395b0ef557ca45afa78c2dd069ef Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Thu, 9 Jan 2025 20:11:26 +0100 Subject: [PATCH 597/856] fix typo with invalid random number of tables Apply fix from https://github.com/sqlancer/sqlancer/pull/1058 --- src/sqlancer/clickhouse/ClickHouseProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseProvider.java b/src/sqlancer/clickhouse/ClickHouseProvider.java index 86f6fd392..c611d77d7 100644 --- a/src/sqlancer/clickhouse/ClickHouseProvider.java +++ b/src/sqlancer/clickhouse/ClickHouseProvider.java @@ -87,7 +87,7 @@ protected ClickHouseSchema readSchema() throws SQLException { @Override public void generateDatabase(ClickHouseGlobalState globalState) throws Exception { - for (int i = 0; i < Randomly.fromOptions(5); i++) { + for (int i = 0; i < Randomly.fromOptions(1, 2, 3, 4, 5); i++) { boolean success; do { String tableName = ClickHouseCommon.createTableName(i); From 0fb45a2242c2d3037e329b59cde76654bc952493 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 15:47:28 +0800 Subject: [PATCH 598/856] Improve the documentation on the currently supported approaches --- README.md | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 27f9579e1..12a879dd7 100644 --- a/README.md +++ b/README.md @@ -67,21 +67,19 @@ Usage: SQLancer [options] [command] [command options] **Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. -# Testing Approaches - -| Approach | Description | -|------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Pivoted Query Synthesis (PQS) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described [here](https://arxiv.org/abs/2001.04174). PQS is the most powerful technique, but also requires more implementation effort than the other two techniques. It is currently unmaintained. | -| Non-optimizing Reference Engine Construction (NoREC) | NoREC aims to find optimization bugs. It is described [here](https://www.manuelrigger.at/preprints/NoREC.pdf). It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. | -| Ternary Logic Partitioning (TLP) | TLP partitions a query into three partitioning queries, whose results are composed and compare to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. | -| Cardinality Estimation Restriction Testing (CERT) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. It is described [here](https://arxiv.org/abs/2306.00355). It derives a query to a more restrict query, whose estimated cardinality should be no more than that for the original query. An violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. | -| Differential Query Plans (DQP) | DQP aims to find logic bugs in database systems by checking whether the query plans of the same query perform consistently. It is described [here](https://bajinsheng.github.io/assets/pdf/dqp_sigmod24.pdf). DQP supports MySQL, MariaDB, and TiDB.| - -# Generation Approaches -| Approach | Description | -|----------|-------------| -| Random Generation | Random generation is the default test case generation approach in SQLancer. First, random tables are generated. Then queries are randomly generated based on the schemas of the tables. | -| Query Plan Guidance (QPG) | QPG is a test case generation method guided by query plan coverage. Given a database state, we mutate it after no new unique query plans have been observed by randomly-generated queries on the database state aiming to cover more unique query plans for exposing more logics of DBMSs. This approach is enabled by option `--qpg-enable` and now supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. | +# Approaches and Papers + +SQLancer has pioneered and includes multiple approaches for DBMS testing, as outlined below in chronological order. + +| Technique | Venue | Links | Description | +|-----------------------------------------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Pivoted Query Synthesis (PQS) | OSDI 2020 | [Paper](https://www.usenix.org/system/files/osdi20-rigger.pdf) [Video](https://www.youtube.com/watch?v=0aeDyXgzo04 ) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described here. PQS effectively detects bugs, but requires more implementation effort than other testing approaches that follow a metamorphic testing or differential testing methodology. Thus, it is currently unmaintained. | +| Non-optimizing Reference Engine Construction (NoREC) | ESEC/FSE 2020 | [Paper](https://arxiv.org/abs/2007.08292) [Video](https://www.youtube.com/watch?v=4mbzytrWJhQ) | NoREC aims to find optimization bugs. It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. The approach applies primarily to simple queries with a filter predicate. | +| Ternary Logic Partitioning (TLP) | OOPSLA 2020 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3428279) [Video](https://www.youtube.com/watch?v=FN9OLbGh0VI) | TLP partitions a query into three partitioning queries, whose results are composed and compared to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. It is among the most widely adopted testing techniques. | +| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | DQP is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | +| Cardinality Estimation Restriction Testing (CERT) | ICSE 2024 | [Paper](https://arxiv.org/pdf/2306.00355) [Code](https://github.com/sqlancer/sqlancer/issues/822) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. From a given input query, it derives a more restrictive query, whose estimated cardinality should be no more than that of the original query. A violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. CERT is the only test oracle that is part of SQLancer that was designed to find performance issues. | +| Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | +| Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | Please find the `.bib` entries [here](docs/PAPERS.md). @@ -139,12 +137,14 @@ Another reason for many failures on a supported version could be that error mess For some DBMSs, SQLancer expects that a database "test" exists, which it then uses as an initial database to connect to. If you have not yet created such a database, you can use a command such as `CREATE DATABASE test` to create this database (e.g., see the [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-createdatabase.html)). -# Additional Documentation +# Links + +Documentation: * [Contributing to SQLancer](CONTRIBUTING.md) * [Papers and .bib entries](docs/PAPERS.md) -# Additional Resources +Other resources: * A talk on Ternary Logic Partitioning (TLP) and SQLancer is available on [YouTube](https://www.youtube.com/watch?v=Np46NQ6lqP8). * An (older) Pivoted Query Synthesis (PQS) talk is available on [YouTube](https://www.youtube.com/watch?v=yzENTaWe7qg). From b6f81e3f5ac14e9b9aef4a37bb06d2cb5fb254ba Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:09:28 +0800 Subject: [PATCH 599/856] Update the resource page --- README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 12a879dd7..0fa2da8e5 100644 --- a/README.md +++ b/README.md @@ -139,14 +139,19 @@ For some DBMSs, SQLancer expects that a database "test" exists, which it then us # Links -Documentation: +Documentation and resources: * [Contributing to SQLancer](CONTRIBUTING.md) * [Papers and .bib entries](docs/PAPERS.md) +* More information on our DBMS testing efforts and the bugs we found is available [here](https://www.manuelrigger.at/dbms-bugs/). -Other resources: +Videos: +* [SQLancer Tutorial Playlist](https://www.youtube.com/playlist?list=PLm7ofmclym1E2LwBeSer_AAhzBSxBYDci) +* [SQLancer Talks](https://youtube.com/playlist?list=PLm7ofmclym1E9-AbYy-PkrMfHpB9VdlZJ) -* A talk on Ternary Logic Partitioning (TLP) and SQLancer is available on [YouTube](https://www.youtube.com/watch?v=Np46NQ6lqP8). -* An (older) Pivoted Query Synthesis (PQS) talk is available on [YouTube](https://www.youtube.com/watch?v=yzENTaWe7qg). -* PingCAP has implemented PQS, NoREC, and TLP in a tool called [go-sqlancer](https://github.com/chaos-mesh/go-sqlancer). -* More information on our DBMS testing efforts and the bugs we found is available [here](https://www.manuelrigger.at/dbms-bugs/). +Closely related tools: +* [go-sqlancer](https://github.com/chaos-mesh/go-sqlancer): re-implementation of some of SQLancer's approaches in Go by PingCAP +* [Jepsen](https://github.com/jepsen-io): testing of distributed (database) systems +* [SQLRight](https://github.com/PSU-Security-Universe/sqlright): coverage-guided DBMS fuzzer, also supporting NoREC and TLP +* [SQLsmith](https://github.com/anse1/sqlsmith): random SQL query generator used for fuzzing +* [Squirrel](https://github.com/s3team/Squirrel): coverage-guided DBMS fuzzer \ No newline at end of file From 324f01a9f9b74f23e374751290799f2045ef658e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:12:39 +0800 Subject: [PATCH 600/856] Improve the FAQ formatting --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0fa2da8e5..18314271e 100644 --- a/README.md +++ b/README.md @@ -127,15 +127,11 @@ We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_inv # FAQ -## I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors? - -In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) file to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). +**I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors?** In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) file to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). Another reason for many failures on a supported version could be that error messages are printed in a non-English locale (which would then be visible in the stack trace). In such a case, try setting the DBMS' locale to English (e.g., see the [PostgreSQL homepage](https://www.postgresql.org/docs/current/locale.html)). -## When starting SQLancer, I get an error such as "database 'test' does not exist". How can I run SQLancer without this error? - -For some DBMSs, SQLancer expects that a database "test" exists, which it then uses as an initial database to connect to. If you have not yet created such a database, you can use a command such as `CREATE DATABASE test` to create this database (e.g., see the [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-createdatabase.html)). +**When starting SQLancer, I get an error such as "database 'test' does not exist". How can I run SQLancer without this error?** For some DBMSs, SQLancer expects that a database "test" exists, which it then uses as an initial database to connect to. If you have not yet created such a database, you can use a command such as `CREATE DATABASE test` to create this database (e.g., see the [PostgreSQL documentation](https://www.postgresql.org/docs/current/sql-createdatabase.html)). # Links From 06a28e929edb2dd5429a278009a88a87219181b5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:21:02 +0800 Subject: [PATCH 601/856] Work on the community part --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 18314271e..c18ca202c 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ SQLancer tackles two essential challenges when automatically testing the DBMSs: 1. **Test input generation**: SQLancer implements approaches for automatically generating SQL statements. It contains various hand-written SQL generators that operate in multiple phases. First, a database schema is created, which refers to a set of tables and their columns. Then, data is inserted into these tables, along with creating various other kinds of database states such as indexes, views, or database-specific options. Finally, queries are generated, which can be validated using one of multiple result validators (also called *test oracles*) that SQLancer provides. Besides the standard approach of creating the statements in an unguided way, SQLancer also supports a test input-generation approach that is feedback-guided and aims to exercise as many unique query plans as possible based on the intuition that doing so would exercise many interesting behaviors in the database system [[ICSE '23]](https://arxiv.org/pdf/2312.17510). 2. **Test oracles**: A key innovation in SQLancer is that it provides ways to find deep kinds of bugs in DBMSs. As a main focus, it can find logic bugs, which are bugs that cause the DBMS to fetch an incorrect result set (e.g., by omitting a record). We have proposed multiple complementary test oracles such as *Ternary Logic Partitioning (TLP)* [[OOPSLA '20]](https://dl.acm.org/doi/pdf/10.1145/3428279), *Non-optimizing Reference Engine Construction (NoREC)* [[ESEC/FSE 2020]](https://arxiv.org/abs/2007.08292), *Pivoted Query Synthesis (PQS)* [[OSDI '20]](https://www.usenix.org/system/files/osdi20-rigger.pdf), *Differential Query Plans (DQP)* [[SIGMOD '24]](https://dl.acm.org/doi/pdf/10.1145/3654991), and *Constant Optimization Driven Database System Testing (CODDTest)* [SIGMOD '25]. It can also find specific categories of performance issues, which refer to cases where a DBMS could reasonably be expected to produce its result more efficiently using a technique called *Cardinality Estimation Restriction Testing (CERT)* [[ICSE '24]](https://arxiv.org/pdf/2306.00355). SQLancer can detect unexpected internal errors (e.g., an error that the database is corrupted) by declaring all potential errors that might be returned by a DBMS for a given query. Finally, SQLancer can find crash bugs, which are bugs that cause the DBMS process to terminate. For this, it uses an implicit test oracle. +**Community.** We have a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. Previously, SQLancer had an account on Twitter/X [@sqlancer_dbms](https://twitter.com/sqlancer_dbms), which is no longer maintained. We have a [blog](https://sqlancer.github.io/posts/), which, as of now, contains only posts by contributors of the [Google Summer of Code project](https://summerofcode.withgoogle.com/archive/2023/organizations/sqlancer). + # Getting Started Minimum Requirements: @@ -120,11 +122,6 @@ Some DBMS were once supported but subsequently removed. | MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | | StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. | - -# Community - -We have created a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. SQLancer's official Twitter handle is [@sqlancer_dbms](https://twitter.com/sqlancer_dbms). - # FAQ **I am running SQLancer on the latest version of a supported DBMS. Is it expected that SQLancer prints many AssertionErrors?** In many cases, SQLancer does not support the latest version of a DBMS. You can check the [`.github/workflows/main.yml`](https://github.com/sqlancer/sqlancer/blob/master/.github/workflows/main.yml) file to determine which version we use in our CI tests, which corresponds to the currently supported version of that DBMS. SQLancer should print only an `AssertionError` and produce a corresponding log file, if it has identified a bug. To upgrade SQLancer to support a new DBMS version, either two options are advisable: (1) the generators can be updated to no longer generate certain patterns that might cause errors (e.g., which might be the case if a keyword or option is no longer supported) or (2) the newly-appearing errors can be added as [expected errors](https://github.com/sqlancer/sqlancer/blob/354d591cfcd37fa1de85ec77ec933d5d975e947a/src/sqlancer/common/query/ExpectedErrors.java) so that SQLancer ignores them when they appear (e.g., this is useful if some error-inducing patterns cannot easily be avoided). From 48052e3c763286155a7aa6d8cc9073bb6846ab24 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:32:33 +0800 Subject: [PATCH 602/856] Move the list of supported DBMSs to CONTRIBUTING.md --- CONTRIBUTING.md | 38 ++++++++++++++++++++++++++++++++++++++ README.md | 39 +-------------------------------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f2217b1f5..85249f315 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,6 +46,43 @@ For a permissive DBMS, implementing the expression generator is easier, since th For a strict DBMS, the better approach is typically to attempt to generate expressions of the expected type. For PostgreSQL, the expression generator thus expects an additional type argument (see [PostgreSQLExpressionGenerator](https://github.com/sqlancer/sqlancer/blob/86647df8aa2dd8d167b5c3ce3297290f5b0b2bcd/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java#L251)). This type is propagated recursively. For example, if we require a predicate for the `WHERE` clause, we pass boolean as a type. The expression generator then calls a method `generateBooleanExpression` that attempts to produce a boolean expression, by, for example, generating a comparison (e.g., `<=`). For the comparison's operands, a random type is then selected and propagated. For example, if an integer type is selected, then `generateExpression` is called with this type once for the left operand, and once for the right operand. Note that this process does not guarantee that the expression will indeed have the expected type. It might happen, for example, that the expression generator attempts to produce an integer value, but that it produces a double value instead, namely when an integer overflow occurs, which, depending on the DBMS, implicitly converts the result to a floating-point value. +#### Supported DBMS + +Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. + +| DBMS | Status | Expression Generation | Description | +| ---------------------------- | ----------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | +| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | +| PostgreSQL | Working | Typed | | +| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | +| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | +| CockroachDB | Working | Typed | | +| TiDB | Working | Untyped | | +| DuckDB | Working | Untyped, Generic | | +| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | +| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | +| OceanBase | Working | Untyped | | +| YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | +| Databend | Working | Typed | | +| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | +| CnosDB | Working | Typed | The implementation of CnosDB currently uses Restful API. | +| Materialize | Working | Typed | | +| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | +| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | +| DataFusion | Preliminary | Typed | Only basic SQL features are supported. | + +#### Previously Supported DBMS + +Some DBMS were once supported but subsequently removed. + +| DBMS | Pull Request | Description | +| ---------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ArangoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because ArangoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | +| StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. + ### Unfixed Bugs Often, some bugs are fixed only after an extended period, meaning that SQLancer will repeatedly report the same bug. In such cases, it might be possible to avoid generating the problematic pattern, or adding an expected error with the internal error message. Rather than, for example, commenting out the code with the bug-inducing pattern, a pattern implemented by the [TiDBBugs class](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java) should be applied. The core idea is to use a public, static flag for each issue, which is set to true as long as the issue persists (e.g., see [bug35652](https://github.com/sqlancer/sqlancer/blob/4c20a94b3ad2c037e1a66c0b637184f8c20faa7e/src/sqlancer/tidb/TiDBBugs.java#L55)). The work-around code is then executed—or the problematic pattern should not be generated—if the flag is set to true (e.g., [an expected error is added for bug35652](https://github.com/sqlancer/sqlancer/blob/59564d818d991d54b32fa5a79c9f733799c090f2/src/sqlancer/tidb/TiDBErrors.java#L47)). This makes it easy to later on identify and remove all such work-around code once the issue has been fixed. @@ -109,3 +146,4 @@ Please pay attention to good commit messages (in particular subject lines). As b 3. Use the imperative mood in the subject line. For example, write "Refactor the handling of indexes" rather than "Refactoring" or "Refactor**ed** the handling of indexes". Please also pay attention to a clean commit history. Rather than merging with the main branch, use `git rebase` to rebase your commits on the main branch. Sometimes, it might happen that you discover an issue only after having already created a commit, for example, when an issue is found by `mvn verify` in the CI checks. Do not introduce a separate commit for such issues. If the issue was introduced by the last commit, you can fix the issue, and use `git commit --amend` to change the latest commit. If the change was introduced by one of the previous commits, you can use `git rebase -i` to change the respective commit. If you already have a number of such commits, you can use `git squash` to "collapse" multiple commits into one. For more information, you might want to read [How (and Why!) to Keep Your Git Commit History Clean](https://about.gitlab.com/blog/2018/06/07/keeping-git-commit-history-clean/) written by Kushal Pandya. + diff --git a/README.md b/README.md index c18ca202c..cfb99d7e2 100644 --- a/README.md +++ b/README.md @@ -83,44 +83,7 @@ SQLancer has pioneered and includes multiple approaches for DBMS testing, as out | Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | | Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | -Please find the `.bib` entries [here](docs/PAPERS.md). - -# Supported DBMS - -Since SQL dialects differ widely, each DBMS to be tested requires a separate implementation. - -| DBMS | Status | Expression Generation | Description | -| ---------------------------- | ----------- | ---------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| SQLite | Working | Untyped | This implementation is currently affected by a significant performance regression that still needs to be investigated | -| MySQL | Working | Untyped | Running this implementation likely uncovers additional, unreported bugs. | -| PostgreSQL | Working | Typed | | -| Citus (PostgreSQL Extension) | Working | Typed | This implementation extends the PostgreSQL implementation of SQLancer, and was contributed by the Citus team. | -| MariaDB | Preliminary | Untyped | The implementation of this DBMS is very preliminary, since we stopped extending it after all but one of our bug reports were addressed. Running it likely uncovers additional, unreported bugs. | -| CockroachDB | Working | Typed | | -| TiDB | Working | Untyped | | -| DuckDB | Working | Untyped, Generic | | -| ClickHouse | Preliminary | Untyped, Generic | Implementing the different table engines was not convenient, which is why only a very preliminary implementation exists. | -| TDEngine | Removed | Untyped | We removed the TDEngine implementation since all but one of our bug reports were still unaddressed five months after we reported them. | -| OceanBase | Working | Untyped | | -| YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | -| Databend | Working | Typed | | -| QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | -| CnosDB | Working | Typed | The implementation of CnosDB currently uses Restful API. | -| Materialize | Working | Typed | | -| Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | -| Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | -| DataFusion | Preliminary | Typed | Only basic SQL features are supported. | - -## Previously Supported DBMS - -Some DBMS were once supported but subsequently removed. - -| DBMS | Pull Request | Description | -| ---------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -| ArangoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because ArangoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | -| Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | -| MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | -| StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. | +Please find the `.bib` entries [here](docs/PAPERS.md). | # FAQ From ef49c7f1b0e980a3bb8253817566ac5a5b577687 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:40:59 +0800 Subject: [PATCH 603/856] Add a brief overview of the supported DBMSs in the main README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index cfb99d7e2..039d1e245 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,8 @@ Usage: SQLancer [options] [command] [command options] **Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. +**Supported DBMSs.** SQLancer requires DBMS-specific code for each DBMS that it supports. As of January 2025, it provides support for Citus, ClickHouse, CnosDB, CockroachDB, Databend, (Apache) DataFusion, (Apache) Doris, DuckDB, H2, HSQLDB, MariaDB, Materialize, MySQL, OceanBase, PostgreSQL, Presto, QuestDB, SQLite3, TiDB, and YugabyteDB. The extent to which the individual DBMSs are supported [differs](https://github.com/sqlancer/sqlancer/blob/documentation-approaches/CONTRIBUTING.md). + # Approaches and Papers SQLancer has pioneered and includes multiple approaches for DBMS testing, as outlined below in chronological order. From 30eb75f1e0d50d85179ff3e297dd50dbc46e1e11 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 16:48:10 +0800 Subject: [PATCH 604/856] Add tutorial links --- CONTRIBUTING.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 85249f315..5db3bdc69 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Development -## Working with Eclipse +## Working with Eclipse [[Video Guide]](https://www.youtube.com/watch?v=KsuGrOLKb9Q) Developing SQLancer using Eclipse is expected to work well. You can import SQLancer with a single step: diff --git a/README.md b/README.md index 039d1e245..ce04e254a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ SQLancer tackles two essential challenges when automatically testing the DBMSs: **Community.** We have a [Slack workspace](https://join.slack.com/t/sqlancer/shared_invite/zt-eozrcao4-ieG29w1LNaBDMF7OB_~ACg) to discuss SQLancer, and DBMS testing in general. Previously, SQLancer had an account on Twitter/X [@sqlancer_dbms](https://twitter.com/sqlancer_dbms), which is no longer maintained. We have a [blog](https://sqlancer.github.io/posts/), which, as of now, contains only posts by contributors of the [Google Summer of Code project](https://summerofcode.withgoogle.com/archive/2023/organizations/sqlancer). -# Getting Started +# Getting Started [[Video Guide]](https://www.youtube.com/watch?v=lcZ6LixPH1Y) Minimum Requirements: * Java 11 or above From 702367f0d301baf304a70a8f3c4ed5c5f1bdd3d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Jan 2025 09:01:57 +0000 Subject: [PATCH 605/856] Bump org.duckdb:duckdb_jdbc from 1.1.0 to 1.1.3 Bumps [org.duckdb:duckdb_jdbc](https://github.com/duckdb/duckdb) from 1.1.0 to 1.1.3. - [Release notes](https://github.com/duckdb/duckdb/releases) - [Commits](https://github.com/duckdb/duckdb/compare/v1.1.0...v1.1.3) --- updated-dependencies: - dependency-name: org.duckdb:duckdb_jdbc dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1bcf92957..110f89fc4 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 1.1.0 + 1.1.3 com.facebook.presto From 72f73677471707fc632793a5585d1abcbced6a69 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 17:03:55 +0800 Subject: [PATCH 606/856] Remove workaround code for a fixed DuckDB bug --- src/sqlancer/duckdb/DuckDBBugs.java | 3 --- src/sqlancer/duckdb/DuckDBErrors.java | 6 ------ 2 files changed, 9 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBBugs.java b/src/sqlancer/duckdb/DuckDBBugs.java index 001deec80..c91661cb5 100644 --- a/src/sqlancer/duckdb/DuckDBBugs.java +++ b/src/sqlancer/duckdb/DuckDBBugs.java @@ -2,9 +2,6 @@ public final class DuckDBBugs { - // https://github.com/duckdb/duckdb/issues/13933 - public static boolean bug13933 = true; - private DuckDBBugs() { } diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index bea136670..5141ca0b1 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -57,12 +57,6 @@ public static List getExpressionErrors() { errors.add("Cannot subtract infinite timestamps"); errors.add("Timestamp difference is out of bounds"); - if (DuckDBBugs.bug13933) { - errors.add("INTERNAL Error: Unimplemented type for normalify"); - errors.add( - "FATAL Error: Failed: database has been invalidated because of a previous fatal error. The database must be restarted prior to being used again"); - } - return errors; } From c677872349904ba9c64ee06531795559939e184c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Jan 2025 00:30:55 +0800 Subject: [PATCH 607/856] Update the H2 version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 110f89fc4..2c616138a 100644 --- a/pom.xml +++ b/pom.xml @@ -340,7 +340,7 @@ com.h2database h2 - 2.1.214 + 2.3.232 org.mongodb From f8101f6299f5e8d53bab622d8e02d73d3475e9a8 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 8 Jan 2025 18:46:04 +0800 Subject: [PATCH 608/856] [MySQL] Support aggregation operations --- .../mysql/MySQLExpectedValueVisitor.java | 7 +++ src/sqlancer/mysql/MySQLToStringVisitor.java | 13 +++++ src/sqlancer/mysql/MySQLVisitor.java | 5 ++ src/sqlancer/mysql/ast/MySQLAggregate.java | 47 +++++++++++++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 8 ++++ 5 files changed, 80 insertions(+) create mode 100644 src/sqlancer/mysql/ast/MySQLAggregate.java diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index 0ff8e389d..29143884f 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -1,6 +1,7 @@ package sqlancer.mysql; import sqlancer.IgnoreMeException; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -166,4 +167,10 @@ public void visit(MySQLText text) { print(text); } + @Override + public void visit(MySQLAggregate aggr) { + print(aggr); + visit(aggr.getExpr()); + } + } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index c4bce3559..bd5a80a58 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -5,6 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -25,6 +26,7 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; public class MySQLToStringVisitor extends ToStringVisitor implements MySQLVisitor { @@ -322,4 +324,15 @@ public void visit(MySQLJoin join) { public void visit(MySQLText text) { sb.append(text.getText()); } + + @Override + public void visit(MySQLAggregate aggr) { + MySQLAggregateFunction func = aggr.getFunc(); + + sb.append(func); + sb.append("("); + sb.append(func.getRandomOption()); + visit(aggr.getExpr()); + sb.append(")"); + } } diff --git a/src/sqlancer/mysql/MySQLVisitor.java b/src/sqlancer/mysql/MySQLVisitor.java index 7b8944f28..e413f7668 100644 --- a/src/sqlancer/mysql/MySQLVisitor.java +++ b/src/sqlancer/mysql/MySQLVisitor.java @@ -1,5 +1,6 @@ package sqlancer.mysql; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -58,6 +59,8 @@ public interface MySQLVisitor { void visit(MySQLText text); + void visit(MySQLAggregate aggregate); + default void visit(MySQLExpression expr) { if (expr instanceof MySQLConstant) { visit((MySQLConstant) expr); @@ -95,6 +98,8 @@ default void visit(MySQLExpression expr) { visit((MySQLCollate) expr); } else if (expr instanceof MySQLText) { visit((MySQLText) expr); + } else if (expr instanceof MySQLAggregate) { + visit((MySQLAggregate) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java new file mode 100644 index 000000000..ad158fd8e --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -0,0 +1,47 @@ +package sqlancer.mysql.ast; + +import sqlancer.Randomly; + +public class MySQLAggregate implements MySQLExpression { + + public enum MySQLAggregateFunction { + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_count. + COUNT("DISTINCT"), + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_sum. + SUM("DISTINCT"), + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_min. + MIN("DISTINCT"), + // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_max. + MAX("DISTINCT"); + + private final String[] options; + + private MySQLAggregateFunction(String... options) { + this.options = options.clone(); + } + + public String getRandomOption() { + if (options.length == 0 || Randomly.getBoolean()) { + return ""; + } + + return Randomly.fromOptions(options); + } + } + + private final MySQLExpression expr; + private final MySQLAggregateFunction func; + + public MySQLAggregate(MySQLExpression expr, MySQLAggregateFunction func) { + this.expr = expr; + this.func = func; + } + + public MySQLExpression getExpr() { + return expr; + } + + public MySQLAggregateFunction getFunc() { + return func; + } +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 0958cca8a..9aeccc858 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -16,6 +16,7 @@ import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; @@ -40,6 +41,7 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; public class MySQLExpressionGenerator extends UntypedExpressionGenerator @@ -248,6 +250,12 @@ public String generateExplainQuery(MySQLSelect select) { return "EXPLAIN " + select.asString(); } + public MySQLAggregate generateAggregate() { + MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); + MySQLExpression expr = generateExpression(); + return new MySQLAggregate(expr, func); + } + @Override public boolean mutate(MySQLSelect select) { List> mutators = new ArrayList<>(); From 66e8bebcb300949556df15884ca10afd33e27c20 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 8 Jan 2025 18:48:14 +0800 Subject: [PATCH 609/856] [MySQL] Generate aggregates in query --- .../gen/MySQLRandomQuerySynthesizer.java | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java index e1a7da75f..6f2c2a656 100644 --- a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -21,11 +21,22 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) MySQLTables tables = globalState.getSchema().getRandomTableNonEmptyTables(); MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(tables.getColumns()); MySQLSelect select = new MySQLSelect(); - List columns = new ArrayList<>(); + + List allColumns = new ArrayList<>(); + List columnsWithoutAggregations = new ArrayList<>(); select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); - columns.addAll(gen.generateExpressions(nrColumns)); - select.setFetchColumns(columns); + for (int i = 0; i < nrColumns; i++) { + if (Randomly.getBoolean()) { + MySQLExpression expression = gen.generateExpression(); + allColumns.add(expression); + columnsWithoutAggregations.add(expression); + } else { + allColumns.add(gen.generateAggregate()); + } + } + select.setFetchColumns(allColumns); + List tableList = tables.getTables().stream().map(t -> new MySQLTableReference(t)) .collect(Collectors.toList()); select.setFromList(tableList); @@ -36,7 +47,7 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) select.setOrderByClauses(gen.generateOrderBys()); } if (Randomly.getBoolean()) { - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); + select.setGroupByExpressions(columnsWithoutAggregations); if (Randomly.getBoolean()) { select.setHavingClause(gen.generateHavingClause()); } From 760bc7a7717517d7d732930613624683b8ff6942 Mon Sep 17 00:00:00 2001 From: baishen Date: Wed, 15 Jan 2025 14:45:17 +0800 Subject: [PATCH 610/856] Databend support Date and Timestamp type --- src/sqlancer/databend/DatabendSchema.java | 32 +++++++++++-------- .../databend/ast/DatabendConstant.java | 4 +++ .../gen/DatabendNewExpressionGenerator.java | 6 ++++ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index 1400d9d5e..af9a3e0ce 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -29,8 +29,7 @@ public class DatabendSchema extends AbstractSchema Date: Wed, 15 Jan 2025 15:49:53 +0800 Subject: [PATCH 611/856] [MySQL] Test aggregate generation --- pom.xml | 6 +++ .../mysql/MySQLExpectedValueVisitor.java | 2 - src/sqlancer/mysql/MySQLToStringVisitor.java | 6 ++- src/sqlancer/mysql/ast/MySQLAggregate.java | 22 ++++++----- .../mysql/gen/MySQLExpressionGenerator.java | 8 +++- .../mysql/MySQLToStringVisitorTest.java | 39 +++++++++++++++++++ 6 files changed, 69 insertions(+), 14 deletions(-) create mode 100644 test/sqlancer/mysql/MySQLToStringVisitorTest.java diff --git a/pom.xml b/pom.xml index 2c616138a..1c7214ad5 100644 --- a/pom.xml +++ b/pom.xml @@ -327,6 +327,12 @@ 5.11.2 test + + org.junit.jupiter + junit-jupiter-params + 5.11.2 + test + org.slf4j slf4j-simple diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index 29143884f..c2bea714b 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -169,8 +169,6 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { - print(aggr); - visit(aggr.getExpr()); } } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index bd5a80a58..892286f87 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -328,10 +328,14 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { MySQLAggregateFunction func = aggr.getFunc(); + String option = aggr.getOption(); sb.append(func); sb.append("("); - sb.append(func.getRandomOption()); + if (option != null) { + sb.append(option); + sb.append(" "); + } visit(aggr.getExpr()); sb.append(")"); } diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java index ad158fd8e..911b9ddd3 100644 --- a/src/sqlancer/mysql/ast/MySQLAggregate.java +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -1,6 +1,6 @@ package sqlancer.mysql.ast; -import sqlancer.Randomly; +import java.util.List; public class MySQLAggregate implements MySQLExpression { @@ -14,27 +14,25 @@ public enum MySQLAggregateFunction { // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_max. MAX("DISTINCT"); - private final String[] options; + private final List options; private MySQLAggregateFunction(String... options) { - this.options = options.clone(); + this.options = List.of(options); } - public String getRandomOption() { - if (options.length == 0 || Randomly.getBoolean()) { - return ""; - } - - return Randomly.fromOptions(options); + public List getOptions() { + return options; } } private final MySQLExpression expr; private final MySQLAggregateFunction func; + private final String option; - public MySQLAggregate(MySQLExpression expr, MySQLAggregateFunction func) { + public MySQLAggregate(MySQLExpression expr, MySQLAggregateFunction func, String option) { this.expr = expr; this.func = func; + this.option = option; } public MySQLExpression getExpr() { @@ -44,4 +42,8 @@ public MySQLExpression getExpr() { public MySQLAggregateFunction getFunc() { return func; } + + public String getOption() { + return option; + } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9aeccc858..a35ee0272 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -253,7 +253,13 @@ public String generateExplainQuery(MySQLSelect select) { public MySQLAggregate generateAggregate() { MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); MySQLExpression expr = generateExpression(); - return new MySQLAggregate(expr, func); + + if (Randomly.getBoolean() && func.getOptions().size() > 0) { + String option = Randomly.fromList(func.getOptions()); + return new MySQLAggregate(expr, func, option); + } else { + return new MySQLAggregate(expr, func, null); + } } @Override diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java new file mode 100644 index 000000000..8a335636c --- /dev/null +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -0,0 +1,39 @@ +package sqlancer.mysql; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLColumnReference; + +public class MySQLToStringVisitorTest { + + @ParameterizedTest + @EnumSource(MySQLAggregate.MySQLAggregateFunction.class) + void visitAggregateWithOptions(MySQLAggregate.MySQLAggregateFunction function) { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", + MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); + + MySQLToStringVisitor visitor = new MySQLToStringVisitor(); + + for (String option : function.getOptions()) { + visitor.visit(new MySQLAggregate(aRef, function, option)); + assertEquals(String.format("%s(%s a)", function, option), visitor.get()); + } + } + + @ParameterizedTest + @EnumSource(MySQLAggregate.MySQLAggregateFunction.class) + void visitAggregateWithoutOptions(MySQLAggregate.MySQLAggregateFunction function) { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", + MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); + + MySQLToStringVisitor visitor = new MySQLToStringVisitor(); + + visitor.visit(new MySQLAggregate(aRef, function, null)); + assertEquals(String.format("%s(a)", function), visitor.get()); + } +} From 7281cc6aa85220aa48ac912e8f8fade8ceb18068 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 15 Jan 2025 16:47:29 +0800 Subject: [PATCH 612/856] [MySQL] Explicate test cases --- pom.xml | 6 -- .../mysql/MySQLExpectedValueVisitor.java | 1 + src/sqlancer/mysql/MySQLToStringVisitor.java | 12 +++- src/sqlancer/mysql/ast/MySQLAggregate.java | 50 +++++++++------ .../mysql/gen/MySQLExpressionGenerator.java | 14 +++-- .../mysql/MySQLToStringVisitorTest.java | 62 ++++++++++++++----- 6 files changed, 94 insertions(+), 51 deletions(-) diff --git a/pom.xml b/pom.xml index 1c7214ad5..2c616138a 100644 --- a/pom.xml +++ b/pom.xml @@ -327,12 +327,6 @@ 5.11.2 test - - org.junit.jupiter - junit-jupiter-params - 5.11.2 - test - org.slf4j slf4j-simple diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index c2bea714b..c9db08306 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -169,6 +169,7 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { + // do nothing } } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 892286f87..7436d7b77 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -328,15 +328,21 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { MySQLAggregateFunction func = aggr.getFunc(); - String option = aggr.getOption(); + String option = func.getOption(); + List exprs = aggr.getExprs(); - sb.append(func); + sb.append(func.getName()); sb.append("("); if (option != null) { sb.append(option); sb.append(" "); } - visit(aggr.getExpr()); + for (int i = 0; i < exprs.size(); i++) { + if (i != 0) { + sb.append(", "); + } + visit(exprs.get(i)); + } sb.append(")"); } } diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java index 911b9ddd3..a5ce2932b 100644 --- a/src/sqlancer/mysql/ast/MySQLAggregate.java +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -6,44 +6,54 @@ public class MySQLAggregate implements MySQLExpression { public enum MySQLAggregateFunction { // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_count. - COUNT("DISTINCT"), + COUNT("COUNT", null, false), + COUNT_DISTINCT("COUNT", "DISTINCT", true), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_sum. - SUM("DISTINCT"), + SUM("SUM", null, false), + SUM_DISTINCT("SUM", "DISTINCT", false), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_min. - MIN("DISTINCT"), + MIN("MIN", null, false), + MIN_DISTINCT("MIN", "DISTINCT", false), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_max. - MAX("DISTINCT"); + MAX("MAX", null, false), + MAX_DISTINCT("MAX", "DISTINCT", false); - private final List options; + private final String name; + private final String option; + private final boolean isVariadic; - private MySQLAggregateFunction(String... options) { - this.options = List.of(options); + private MySQLAggregateFunction(String name, String option, boolean isVariadic) { + this.name = name; + this.option = option; + this.isVariadic = isVariadic; } - public List getOptions() { - return options; + public String getName() { + return this.name; + } + + public String getOption() { + return option; + } + + public boolean isVariadic() { + return this.isVariadic; } } - private final MySQLExpression expr; + private final List exprs; private final MySQLAggregateFunction func; - private final String option; - public MySQLAggregate(MySQLExpression expr, MySQLAggregateFunction func, String option) { - this.expr = expr; + public MySQLAggregate(List exprs, MySQLAggregateFunction func) { + this.exprs = exprs; this.func = func; - this.option = option; } - public MySQLExpression getExpr() { - return expr; + public List getExprs() { + return exprs; } public MySQLAggregateFunction getFunc() { return func; } - - public String getOption() { - return option; - } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index a35ee0272..932119a51 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.IntStream; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -252,13 +253,16 @@ public String generateExplainQuery(MySQLSelect select) { public MySQLAggregate generateAggregate() { MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); - MySQLExpression expr = generateExpression(); - if (Randomly.getBoolean() && func.getOptions().size() > 0) { - String option = Randomly.fromList(func.getOptions()); - return new MySQLAggregate(expr, func, option); + if (func.isVariadic()) { + int nrExprs = Randomly.smallNumber() + 1; + List exprs = IntStream.range(0, nrExprs) + .mapToObj(index -> generateExpression()) + .collect(Collectors.toList()); + + return new MySQLAggregate(exprs, func); } else { - return new MySQLAggregate(expr, func, null); + return new MySQLAggregate(List.of(generateExpression()), func); } } diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 8a335636c..b5a5d1ffb 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -1,39 +1,67 @@ package sqlancer.mysql; import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; + +import java.util.List; + +import org.junit.jupiter.api.Test; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLColumnReference; public class MySQLToStringVisitorTest { - - @ParameterizedTest - @EnumSource(MySQLAggregate.MySQLAggregateFunction.class) - void visitAggregateWithOptions(MySQLAggregate.MySQLAggregateFunction function) { + + @Test + void visitAggregateToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); - MySQLToStringVisitor visitor = new MySQLToStringVisitor(); + MySQLAggregate aggrCount = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT + ); + assertEquals("COUNT(a)", MySQLVisitor.asString(aggrCount)); + + MySQLAggregate aggrSum = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM + ); + assertEquals("SUM(a)", MySQLVisitor.asString(aggrSum)); - for (String option : function.getOptions()) { - visitor.visit(new MySQLAggregate(aRef, function, option)); - assertEquals(String.format("%s(%s a)", function, option), visitor.get()); - } + MySQLAggregate aggrMin = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN + ); + assertEquals("MIN(a)", MySQLVisitor.asString(aggrMin)); + + MySQLAggregate aggrMax = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX + ); + assertEquals("MAX(a)", MySQLVisitor.asString(aggrMax)); } - @ParameterizedTest - @EnumSource(MySQLAggregate.MySQLAggregateFunction.class) - void visitAggregateWithoutOptions(MySQLAggregate.MySQLAggregateFunction function) { + @Test + void visitAggregateWithDistinctToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); - MySQLToStringVisitor visitor = new MySQLToStringVisitor(); + MySQLAggregate aggrCountDistinct = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT_DISTINCT + ); + assertEquals("COUNT(DISTINCT a)", MySQLVisitor.asString(aggrCountDistinct)); + + MySQLAggregate aggrSumDistinct = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM_DISTINCT + ); + assertEquals("SUM(DISTINCT a)", MySQLVisitor.asString(aggrSumDistinct)); + + MySQLAggregate aggrMinDistinct = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN_DISTINCT + ); + assertEquals("MIN(DISTINCT a)", MySQLVisitor.asString(aggrMinDistinct)); - visitor.visit(new MySQLAggregate(aRef, function, null)); - assertEquals(String.format("%s(a)", function), visitor.get()); + MySQLAggregate aggrMaxDistinct = new MySQLAggregate( + List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX_DISTINCT + ); + assertEquals("MAX(DISTINCT a)", MySQLVisitor.asString(aggrMaxDistinct)); } } From 29a2f4d7de1a0f1248c75779a6e5a124a2762461 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 15 Jan 2025 16:54:57 +0800 Subject: [PATCH 613/856] [MySQL] Add group by if aggregate has been generated --- src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java index 6f2c2a656..d701072ee 100644 --- a/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java +++ b/src/sqlancer/mysql/gen/MySQLRandomQuerySynthesizer.java @@ -25,6 +25,8 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) List allColumns = new ArrayList<>(); List columnsWithoutAggregations = new ArrayList<>(); + boolean hasGeneratedAggregate = false; + select.setSelectType(Randomly.fromOptions(MySQLSelect.SelectType.values())); for (int i = 0; i < nrColumns; i++) { if (Randomly.getBoolean()) { @@ -33,6 +35,7 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) columnsWithoutAggregations.add(expression); } else { allColumns.add(gen.generateAggregate()); + hasGeneratedAggregate = true; } } select.setFetchColumns(allColumns); @@ -46,7 +49,7 @@ public static MySQLSelect generate(MySQLGlobalState globalState, int nrColumns) if (Randomly.getBooleanWithRatherLowProbability()) { select.setOrderByClauses(gen.generateOrderBys()); } - if (Randomly.getBoolean()) { + if (hasGeneratedAggregate || Randomly.getBoolean()) { select.setGroupByExpressions(columnsWithoutAggregations); if (Randomly.getBoolean()) { select.setHavingClause(gen.generateHavingClause()); From 910a0b749ab4e88503a2610bcc45ca4d110279eb Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Wed, 15 Jan 2025 19:42:27 +0800 Subject: [PATCH 614/856] Format files --- src/sqlancer/mysql/MySQLToStringVisitor.java | 2 +- src/sqlancer/mysql/ast/MySQLAggregate.java | 16 +++---- .../mysql/gen/MySQLExpressionGenerator.java | 9 ++-- .../mysql/MySQLToStringVisitorTest.java | 42 +++++++------------ 4 files changed, 25 insertions(+), 44 deletions(-) diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 7436d7b77..4d9607717 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -6,6 +6,7 @@ import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -26,7 +27,6 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; -import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; public class MySQLToStringVisitor extends ToStringVisitor implements MySQLVisitor { diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java index a5ce2932b..94c4e426b 100644 --- a/src/sqlancer/mysql/ast/MySQLAggregate.java +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -3,26 +3,22 @@ import java.util.List; public class MySQLAggregate implements MySQLExpression { - + public enum MySQLAggregateFunction { // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_count. - COUNT("COUNT", null, false), - COUNT_DISTINCT("COUNT", "DISTINCT", true), + COUNT("COUNT", null, false), COUNT_DISTINCT("COUNT", "DISTINCT", true), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_sum. - SUM("SUM", null, false), - SUM_DISTINCT("SUM", "DISTINCT", false), + SUM("SUM", null, false), SUM_DISTINCT("SUM", "DISTINCT", false), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_min. - MIN("MIN", null, false), - MIN_DISTINCT("MIN", "DISTINCT", false), + MIN("MIN", null, false), MIN_DISTINCT("MIN", "DISTINCT", false), // See https://dev.mysql.com/doc/refman/8.4/en/aggregate-functions.html#function_max. - MAX("MAX", null, false), - MAX_DISTINCT("MAX", "DISTINCT", false); + MAX("MAX", null, false), MAX_DISTINCT("MAX", "DISTINCT", false); private final String name; private final String option; private final boolean isVariadic; - private MySQLAggregateFunction(String name, String option, boolean isVariadic) { + MySQLAggregateFunction(String name, String option, boolean isVariadic) { this.name = name; this.option = option; this.isVariadic = isVariadic; diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 932119a51..f4b868f61 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -18,6 +18,7 @@ import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; @@ -42,7 +43,6 @@ import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; -import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; public class MySQLExpressionGenerator extends UntypedExpressionGenerator @@ -256,10 +256,9 @@ public MySQLAggregate generateAggregate() { if (func.isVariadic()) { int nrExprs = Randomly.smallNumber() + 1; - List exprs = IntStream.range(0, nrExprs) - .mapToObj(index -> generateExpression()) - .collect(Collectors.toList()); - + List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) + .collect(Collectors.toList()); + return new MySQLAggregate(exprs, func); } else { return new MySQLAggregate(List.of(generateExpression()), func); diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index b5a5d1ffb..5aa9b4e14 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -13,55 +13,41 @@ public class MySQLToStringVisitorTest { @Test void visitAggregateToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", - MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); - MySQLAggregate aggrCount = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT - ); + MySQLAggregate aggrCount = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT); assertEquals("COUNT(a)", MySQLVisitor.asString(aggrCount)); - MySQLAggregate aggrSum = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM - ); + MySQLAggregate aggrSum = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM); assertEquals("SUM(a)", MySQLVisitor.asString(aggrSum)); - MySQLAggregate aggrMin = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN - ); + MySQLAggregate aggrMin = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN); assertEquals("MIN(a)", MySQLVisitor.asString(aggrMin)); - MySQLAggregate aggrMax = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX - ); + MySQLAggregate aggrMax = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX); assertEquals("MAX(a)", MySQLVisitor.asString(aggrMax)); } @Test void visitAggregateWithDistinctToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", - MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); - MySQLAggregate aggrCountDistinct = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT_DISTINCT - ); + MySQLAggregate aggrCountDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.COUNT_DISTINCT); assertEquals("COUNT(DISTINCT a)", MySQLVisitor.asString(aggrCountDistinct)); - MySQLAggregate aggrSumDistinct = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.SUM_DISTINCT - ); + MySQLAggregate aggrSumDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.SUM_DISTINCT); assertEquals("SUM(DISTINCT a)", MySQLVisitor.asString(aggrSumDistinct)); - MySQLAggregate aggrMinDistinct = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MIN_DISTINCT - ); + MySQLAggregate aggrMinDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.MIN_DISTINCT); assertEquals("MIN(DISTINCT a)", MySQLVisitor.asString(aggrMinDistinct)); - MySQLAggregate aggrMaxDistinct = new MySQLAggregate( - List.of(aRef), MySQLAggregate.MySQLAggregateFunction.MAX_DISTINCT - ); + MySQLAggregate aggrMaxDistinct = new MySQLAggregate(List.of(aRef), + MySQLAggregate.MySQLAggregateFunction.MAX_DISTINCT); assertEquals("MAX(DISTINCT a)", MySQLVisitor.asString(aggrMaxDistinct)); } } From f6c1f033259dc1700da6df4c58f5746ab3bc0409 Mon Sep 17 00:00:00 2001 From: baishen Date: Sun, 19 Jan 2025 11:54:09 +0800 Subject: [PATCH 615/856] fix tests --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/ast/DatabendConstant.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e7d69c00a..91188369b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -205,7 +205,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.542-nightly + image: datafuselabs/databend:v1.2.687-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 4c93ac213..142cc3090 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -322,7 +322,8 @@ public static class DatabendDateConstant extends DatabendConstant { public String textRepr; public DatabendDateConstant(long val) { - Timestamp timestamp = new Timestamp(val); + long t = val % 250000000000000l; + Timestamp timestamp = new Timestamp(t); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); textRepr = dateFormat.format(timestamp); } @@ -357,7 +358,8 @@ public static class DatabendTimestampConstant extends DatabendConstant { public String textRepr; public DatabendTimestampConstant(long val) { - Timestamp timestamp = new Timestamp(val); + long t = val % 250000000000000l; + Timestamp timestamp = new Timestamp(t); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); textRepr = dateFormat.format(timestamp); } From d59008dc3134ccdf1158207ca053c1e744accfeb Mon Sep 17 00:00:00 2001 From: baishen Date: Mon, 20 Jan 2025 11:44:57 +0800 Subject: [PATCH 616/856] add comments --- src/sqlancer/databend/ast/DatabendConstant.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index 142cc3090..f5355df58 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -322,7 +322,9 @@ public static class DatabendDateConstant extends DatabendConstant { public String textRepr; public DatabendDateConstant(long val) { - long t = val % 250000000000000l; + // Databend supports `date` type where the year cannot exceed `9999`, + // the value is truncated to ensure generate legitimate `date` value. + long t = val % 253380000000000L; Timestamp timestamp = new Timestamp(t); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); textRepr = dateFormat.format(timestamp); @@ -358,7 +360,9 @@ public static class DatabendTimestampConstant extends DatabendConstant { public String textRepr; public DatabendTimestampConstant(long val) { - long t = val % 250000000000000l; + // Databend supports `timestamp` type where the year cannot exceed `9999`, + // the value is truncated to ensure generate legitimate `timestamp` value. + long t = val % 253380000000000L; Timestamp timestamp = new Timestamp(t); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); textRepr = dateFormat.format(timestamp); From 292a91b230d94ad433118fbfc5caf34e1c952785 Mon Sep 17 00:00:00 2001 From: baishen Date: Mon, 20 Jan 2025 12:24:14 +0800 Subject: [PATCH 617/856] simplify --- src/sqlancer/databend/ast/DatabendConstant.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/databend/ast/DatabendConstant.java b/src/sqlancer/databend/ast/DatabendConstant.java index f5355df58..919942a8a 100644 --- a/src/sqlancer/databend/ast/DatabendConstant.java +++ b/src/sqlancer/databend/ast/DatabendConstant.java @@ -48,6 +48,13 @@ public double asFloat() { throw new UnsupportedOperationException(this.toString()); } + protected Timestamp truncateTimestamp(long val) { + // Databend supports `date` and `timestamp` type where the year cannot exceed `9999`, + // the value is truncated to ensure generate legitimate `date` and `timestamp` value. + long t = val % 253380000000000L; + return new Timestamp(t); + } + public abstract DatabendConstant isEquals(DatabendConstant rightVal); public abstract DatabendConstant isLessThan(DatabendConstant rightVal); @@ -322,10 +329,7 @@ public static class DatabendDateConstant extends DatabendConstant { public String textRepr; public DatabendDateConstant(long val) { - // Databend supports `date` type where the year cannot exceed `9999`, - // the value is truncated to ensure generate legitimate `date` value. - long t = val % 253380000000000L; - Timestamp timestamp = new Timestamp(t); + Timestamp timestamp = truncateTimestamp(val); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); textRepr = dateFormat.format(timestamp); } @@ -360,10 +364,7 @@ public static class DatabendTimestampConstant extends DatabendConstant { public String textRepr; public DatabendTimestampConstant(long val) { - // Databend supports `timestamp` type where the year cannot exceed `9999`, - // the value is truncated to ensure generate legitimate `timestamp` value. - long t = val % 253380000000000L; - Timestamp timestamp = new Timestamp(t); + Timestamp timestamp = truncateTimestamp(val); SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); textRepr = dateFormat.format(timestamp); } From 1458d5e044603aed13514ee2b7368a5fa4fdc1b1 Mon Sep 17 00:00:00 2001 From: MAOXIONGKAI <40223039+MAOXIONGKAI@users.noreply.github.com> Date: Mon, 20 Jan 2025 15:28:52 +0800 Subject: [PATCH 618/856] Fix typo in Approaches and Papers section of REAMD.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ce04e254a..f88b77faa 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ SQLancer has pioneered and includes multiple approaches for DBMS testing, as out | Pivoted Query Synthesis (PQS) | OSDI 2020 | [Paper](https://www.usenix.org/system/files/osdi20-rigger.pdf) [Video](https://www.youtube.com/watch?v=0aeDyXgzo04 ) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described here. PQS effectively detects bugs, but requires more implementation effort than other testing approaches that follow a metamorphic testing or differential testing methodology. Thus, it is currently unmaintained. | | Non-optimizing Reference Engine Construction (NoREC) | ESEC/FSE 2020 | [Paper](https://arxiv.org/abs/2007.08292) [Video](https://www.youtube.com/watch?v=4mbzytrWJhQ) | NoREC aims to find optimization bugs. It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. The approach applies primarily to simple queries with a filter predicate. | | Ternary Logic Partitioning (TLP) | OOPSLA 2020 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3428279) [Video](https://www.youtube.com/watch?v=FN9OLbGh0VI) | TLP partitions a query into three partitioning queries, whose results are composed and compared to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. It is among the most widely adopted testing techniques. | -| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | DQP is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | +| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | QPG is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | | Cardinality Estimation Restriction Testing (CERT) | ICSE 2024 | [Paper](https://arxiv.org/pdf/2306.00355) [Code](https://github.com/sqlancer/sqlancer/issues/822) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. From a given input query, it derives a more restrictive query, whose estimated cardinality should be no more than that of the original query. A violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. CERT is the only test oracle that is part of SQLancer that was designed to find performance issues. | | Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | | Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | @@ -112,4 +112,4 @@ Closely related tools: * [Jepsen](https://github.com/jepsen-io): testing of distributed (database) systems * [SQLRight](https://github.com/PSU-Security-Universe/sqlright): coverage-guided DBMS fuzzer, also supporting NoREC and TLP * [SQLsmith](https://github.com/anse1/sqlsmith): random SQL query generator used for fuzzing -* [Squirrel](https://github.com/s3team/Squirrel): coverage-guided DBMS fuzzer \ No newline at end of file +* [Squirrel](https://github.com/s3team/Squirrel): coverage-guided DBMS fuzzer From 6efee976f4c41bfaf053fd4e4baeab8005181ec6 Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Wed, 5 Feb 2025 10:59:43 +0100 Subject: [PATCH 619/856] Update ClickHouseErrors.java (#1071) * Update ClickHouseErrors.java * format --- src/sqlancer/clickhouse/ClickHouseErrors.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index e6a4613d9..2c4d8d1b8 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -55,7 +55,7 @@ public static List getExpectedExpressionErrors() { // AST generation logic "MULTIPLE_EXPRESSIONS_FOR_ALIAS", "AMBIGUOUS_IDENTIFIER", // https://github.com/ClickHouse/ClickHouse/issues/45389 "AMBIGUOUS_COLUMN_NAME", // same https://github.com/ClickHouse/ClickHouse/issues/45389 - "Cannot parse number with multiple sign"); + "No equality condition found in JOIN ON expression", "Cannot parse number with multiple sign"); } public static void addExpectedExpressionErrors(ExpectedErrors errors) { From c7e4ca220789f1e46ef5cc11e209ba3d3e4ac7a0 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Wed, 5 Feb 2025 21:26:59 +1030 Subject: [PATCH 620/856] Mark support for v13, update test / comment URLs accordingly. As per discussionsin Issue #912 , SQLancer works fine with Postgres v13. This commit updates the test harness to run against Postgres v13 (instead of v12 currently). Also, the documentation links in README & code-comment URLs point to unreleased / outdated postgres documentation versions - For e.g. links were pointing to different postgres version documentations like devel (unreleased) / current / v12 or even v9.5 / v11 (both already EOL). This commit ensures that all www.postgresql.org documentation links in the comments, point to the supported version of postgres (which as of this commit is v13). As and when we advance support for a more recent version, we should update the doc links accordingly too. --- .github/workflows/main.yml | 2 +- src/sqlancer/postgres/PostgresProvider.java | 2 +- .../ast/PostgresFunctionWithUnknownResult.java | 16 ++++++++-------- .../postgres/gen/PostgresSetGenerator.java | 14 +++++++------- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91188369b..b42d40aed 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -467,7 +467,7 @@ jobs: - name: Set up PostgreSQL uses: harmon758/postgresql-action@v1.0.0 with: - postgresql version: '12' + postgresql version: '13' postgresql user: 'sqlancer' postgresql password: 'sqlancer' postgresql db: 'test' diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 7c86dc628..b255bb542 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -109,7 +109,7 @@ public enum Action implements AbstractAction { RESET_ROLE((g) -> new SQLQueryAdapter("RESET ROLE")), // COMMENT_ON(PostgresCommentGenerator::generate), // RESET((g) -> new SQLQueryAdapter("RESET ALL") /* - * https://www.postgresql.org/docs/devel/sql-reset.html TODO: also + * https://www.postgresql.org/docs/13/sql-reset.html TODO: also * configuration parameter */), // NOTIFY(PostgresNotifyGenerator::createNotify), // diff --git a/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java b/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java index 3357a7db5..287f46784 100644 --- a/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java +++ b/src/sqlancer/postgres/ast/PostgresFunctionWithUnknownResult.java @@ -19,11 +19,11 @@ public enum PostgresFunctionWithUnknownResult { TEXT("text", PostgresDataType.TEXT, PostgresDataType.INET), INET_SAME_FAMILY("inet_same_family", PostgresDataType.BOOLEAN, PostgresDataType.INET, PostgresDataType.INET), - // https://www.postgresql.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-SIGNAL-TABLE + // https://www.postgresql.org/docs/13/functions-admin.html#FUNCTIONS-ADMIN-SIGNAL-TABLE // PG_RELOAD_CONF("pg_reload_conf", PostgresDataType.BOOLEAN), // too much output // PG_ROTATE_LOGFILE("pg_rotate_logfile", PostgresDataType.BOOLEAN), prints warning - // https://www.postgresql.org/docs/devel/functions-info.html#FUNCTIONS-INFO-SESSION-TABLE + // https://www.postgresql.org/docs/13/functions-info.html#FUNCTIONS-INFO-SESSION-TABLE CURRENT_DATABASE("current_database", PostgresDataType.TEXT), // name // CURRENT_QUERY("current_query", PostgresDataType.TEXT), // can generate false positives CURRENT_SCHEMA("current_schema", PostgresDataType.TEXT), // name @@ -87,7 +87,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx TO_HEX("to_hex", PostgresDataType.INT, PostgresDataType.TEXT), TRANSLATE("translate", PostgresDataType.TEXT, PostgresDataType.TEXT, PostgresDataType.TEXT, PostgresDataType.TEXT), // mathematical functions - // https://www.postgresql.org/docs/9.5/functions-math.html + // https://www.postgresql.org/docs/13/functions-math.html ABS("abs", PostgresDataType.REAL, PostgresDataType.REAL), CBRT("cbrt", PostgresDataType.REAL, PostgresDataType.REAL), CEILING("ceiling", PostgresDataType.REAL), // DEGREES("degrees", PostgresDataType.REAL), EXP("exp", PostgresDataType.REAL), LN("ln", PostgresDataType.REAL), @@ -98,7 +98,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx FLOOR("floor", PostgresDataType.REAL), // trigonometric functions - complete - // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-TRIG-TABLE + // https://www.postgresql.org/docs/13/functions-math.html#FUNCTIONS-MATH-TRIG-TABLE ACOS("acos", PostgresDataType.REAL), // ACOSD("acosd", PostgresDataType.REAL), // ASIN("asin", PostgresDataType.REAL), // @@ -117,7 +117,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx TAND("tand", PostgresDataType.REAL), // // hyperbolic functions - complete - // https://www.postgresql.org/docs/12/functions-math.html#FUNCTIONS-MATH-HYP-TABLE + // https://www.postgresql.org/docs/13/functions-math.html#FUNCTIONS-MATH-HYP-TABLE SINH("sinh", PostgresDataType.REAL), // COSH("cosh", PostgresDataType.REAL), // TANH("tanh", PostgresDataType.REAL), // @@ -125,12 +125,12 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx ACOSH("acosh", PostgresDataType.REAL), // ATANH("atanh", PostgresDataType.REAL), // - // https://www.postgresql.org/docs/devel/functions-binarystring.html + // https://www.postgresql.org/docs/13/functions-binarystring.html GET_BIT("get_bit", PostgresDataType.INT, PostgresDataType.TEXT, PostgresDataType.INT), GET_BYTE("get_byte", PostgresDataType.INT, PostgresDataType.TEXT, PostgresDataType.INT), // range functions - // https://www.postgresql.org/docs/devel/functions-range.html#RANGE-FUNCTIONS-TABLE + // https://www.postgresql.org/docs/13/functions-range.html#RANGE-FUNCTIONS-TABLE RANGE_LOWER("lower", PostgresDataType.INT, PostgresDataType.RANGE), // RANGE_UPPER("upper", PostgresDataType.INT, PostgresDataType.RANGE), // RANGE_ISEMPTY("isempty", PostgresDataType.BOOLEAN, PostgresDataType.RANGE), // @@ -140,7 +140,7 @@ public PostgresExpression[] getArguments(PostgresDataType returnType, PostgresEx RANGE_UPPER_INF("upper_inf", PostgresDataType.BOOLEAN, PostgresDataType.RANGE), // RANGE_MERGE("range_merge", PostgresDataType.RANGE, PostgresDataType.RANGE, PostgresDataType.RANGE), // - // https://www.postgresql.org/docs/devel/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE + // https://www.postgresql.org/docs/13/functions-admin.html#FUNCTIONS-ADMIN-DBSIZE GET_COLUMN_SIZE("get_column_size", PostgresDataType.INT, PostgresDataType.TEXT); // PG_DATABASE_SIZE("pg_database_size", PostgresDataType.INT, PostgresDataType.INT); // PG_SIZE_BYTES("pg_size_bytes", PostgresDataType.INT, PostgresDataType.TEXT); diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index f600f1f04..440172c4b 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -14,7 +14,7 @@ private PostgresSetGenerator() { } private enum ConfigurationOption { - // https://www.postgresql.org/docs/11/runtime-config-wal.html + // https://www.postgresql.org/docs/13/runtime-config-wal.html // This parameter can only be set at server start. // WAL_LEVEL("wal_level", (r) -> Randomly.fromOptions("replica", "minimal", "logical")), // FSYNC("fsync", (r) -> Randomly.fromOptions(1, 0)), @@ -37,7 +37,7 @@ private enum ConfigurationOption { // archive_mode // archive_command // archive_timeout - // https://www.postgresql.org/docs/11/runtime-config-statistics.html + // https://www.postgresql.org/docs/13/runtime-config-statistics.html // 19.9.1. Query and Index Statistics Collector TRACK_ACTIVITIES("track_activities", (r) -> Randomly.fromOptions(1, 0)), // track_activity_query_size @@ -46,7 +46,7 @@ private enum ConfigurationOption { TRACK_FUNCTIONS("track_functions", (r) -> Randomly.fromOptions("'none'", "'pl'", "'all'")), // stats_temp_directory // TODO 19.9.2. Statistics Monitoring - // https://www.postgresql.org/docs/11/runtime-config-autovacuum.html + // https://www.postgresql.org/docs/13/runtime-config-autovacuum.html // all can only be set at server-conf time // 19.11. Client Connection Defaults VACUUM_FREEZE_TABLE_AGE("vacuum_freeze_table_age", (r) -> Randomly.fromOptions(0, 5, 10, 100, 500, 2000000000)), @@ -60,7 +60,7 @@ private enum ConfigurationOption { // 19.13. Version and Platform Compatibility DEFAULT_WITH_OIDS("default_with_oids", (r) -> Randomly.fromOptions(0, 1)), SYNCHRONIZED_SEQSCANS("synchronize_seqscans", (r) -> Randomly.fromOptions(0, 1)), - // https://www.postgresql.org/docs/devel/runtime-config-query.html + // https://www.postgresql.org/docs/13/runtime-config-query.html ENABLE_BITMAPSCAN("enable_bitmapscan", (r) -> Randomly.fromOptions(1, 0)), ENABLE_GATHERMERGE("enable_gathermerge", (r) -> Randomly.fromOptions(1, 0)), ENABLE_HASHJOIN("enable_hashjoin", (r) -> Randomly.fromOptions(1, 0)), @@ -78,7 +78,7 @@ private enum ConfigurationOption { ENABLE_SORT("enable_sort", (r) -> Randomly.fromOptions(1, 0)), ENABLE_TIDSCAN("enable_tidscan", (r) -> Randomly.fromOptions(1, 0)), // 19.7.2. Planner Cost Constants (complete as of March 2020) - // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-CONSTANTS + // https://www.postgresql.org/docs/13/runtime-config-query.html#RUNTIME-CONFIG-QUERY-CONSTANTS SEQ_PAGE_COST("seq_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), RANDOM_PAGE_COST("random_page_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), CPU_TUPLE_COST("cpu_tuple_cost", (r) -> Randomly.fromOptions(0d, 0.00001, 0.05, 0.1, 1, 10, 10000)), @@ -94,7 +94,7 @@ private enum ConfigurationOption { JIT_OPTIMIZE_ABOVE_COST("jit_optimize_above_cost", (r) -> Randomly.fromOptions(0, r.getLong(-1, Long.MAX_VALUE))), // 19.7.3. Genetic Query Optimizer (complete as of March 2020) - // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-GEQO + // https://www.postgresql.org/docs/13/runtime-config-query.html#RUNTIME-CONFIG-QUERY-GEQO GEQO("geqo", (r) -> Randomly.fromOptions(1, 0)), GEQO_THRESHOLD("geqo_threshold", (r) -> r.getInteger(2, 2147483647)), GEQO_EFFORT("geqo_effort", (r) -> r.getInteger(1, 10)), @@ -103,7 +103,7 @@ private enum ConfigurationOption { GEQO_SELECTION_BIAS("geqo_selection_bias", (r) -> Randomly.fromOptions(1.5, 1.8, 2.0)), GEQO_SEED("geqo_seed", (r) -> Randomly.fromOptions(0, 0.5, 1)), // 19.7.4. Other Planner Options (complete as of March 2020) - // https://www.postgresql.org/docs/current/runtime-config-query.html#RUNTIME-CONFIG-QUERY-OTHER + // https://www.postgresql.org/docs/13/runtime-config-query.html#RUNTIME-CONFIG-QUERY-OTHER DEFAULT_STATISTICS_TARGET("default_statistics_target", (r) -> r.getInteger(1, 10000)), CONSTRAINT_EXCLUSION("constraint_exclusion", (r) -> Randomly.fromOptions("on", "off", "partition")), CURSOR_TUPLE_FRACTION("cursor_tuple_fraction", From 1a9e496eb1e98a90ebf290d333aa1306df3e2fa1 Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Mon, 3 Mar 2025 15:19:25 +0200 Subject: [PATCH 621/856] add postgres explain generator --- .../gen/PostgresExplainGenerator.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 src/sqlancer/postgres/gen/PostgresExplainGenerator.java diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java new file mode 100644 index 000000000..318e5eead --- /dev/null +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -0,0 +1,36 @@ +package sqlancer.postgres.gen; + +import jnr.ffi.Struct; +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresProvider; +import sqlancer.postgres.PostgresProvider.Action; + +public class PostgresExplainGenerator { + + private PostgresExplainGenerator(){ + + } + + public static SQLQueryAdapter explain(PostgresGlobalState globalState) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + Action action; + + do { + action = Randomly.fromOptions(PostgresProvider.Action.values()); + } while (action == Action.EXPLAIN); + SQLQueryAdapter query = action.getQuery(globalState); + sb.append(query); + return new SQLQueryAdapter(sb.toString(),query.getExpectedErrors()); + } + + public static String explain(String selectStr) throws Exception{ + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + sb.append(selectStr); + return sb.toString(); + } + +} From ee8dd08d8a0d459bed4eaa245aa2d727a0e7bb5d Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Mon, 3 Mar 2025 18:45:36 +0200 Subject: [PATCH 622/856] add QPG for postgres --- src/sqlancer/postgres/PostgresProvider.java | 78 ++++++++++++++----- .../gen/PostgresExplainGenerator.java | 1 - .../postgres/gen/PostgresInsertGenerator.java | 4 + 3 files changed, 62 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 7c86dc628..f88c2c65b 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -1,5 +1,6 @@ package sqlancer.postgres; +import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; import java.sql.Connection; @@ -7,6 +8,8 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import com.google.auto.service.AutoService; @@ -22,26 +25,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.gen.PostgresAlterTableGenerator; -import sqlancer.postgres.gen.PostgresAnalyzeGenerator; -import sqlancer.postgres.gen.PostgresClusterGenerator; -import sqlancer.postgres.gen.PostgresCommentGenerator; -import sqlancer.postgres.gen.PostgresDeleteGenerator; -import sqlancer.postgres.gen.PostgresDiscardGenerator; -import sqlancer.postgres.gen.PostgresDropIndexGenerator; -import sqlancer.postgres.gen.PostgresIndexGenerator; -import sqlancer.postgres.gen.PostgresInsertGenerator; -import sqlancer.postgres.gen.PostgresNotifyGenerator; -import sqlancer.postgres.gen.PostgresReindexGenerator; -import sqlancer.postgres.gen.PostgresSequenceGenerator; -import sqlancer.postgres.gen.PostgresSetGenerator; -import sqlancer.postgres.gen.PostgresStatisticsGenerator; -import sqlancer.postgres.gen.PostgresTableGenerator; -import sqlancer.postgres.gen.PostgresTransactionGenerator; -import sqlancer.postgres.gen.PostgresTruncateGenerator; -import sqlancer.postgres.gen.PostgresUpdateGenerator; -import sqlancer.postgres.gen.PostgresVacuumGenerator; -import sqlancer.postgres.gen.PostgresViewGenerator; +import sqlancer.postgres.gen.*; // EXISTS // IN @@ -73,6 +57,7 @@ protected PostgresProvider(Class globalClass, Class { + EXPLAIN(PostgresExplainGenerator::explain), ANALYZE(PostgresAnalyzeGenerator::create), // ALTER_TABLE(g -> PostgresAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g, generateOnlyKnown)), // @@ -146,6 +131,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { nrPerformed = r.getInteger(0, 5); break; case COMMIT: + case EXPLAIN: nrPerformed = r.getInteger(0, 0); break; case ALTER_TABLE: @@ -183,6 +169,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; + default: throw new AssertionError(a); } @@ -349,4 +336,55 @@ public String getDBMSName() { return "postgres"; } + + @Override + public String getQueryPlan(String selectStr, PostgresGlobalState globalState) throws Exception { + String queryPlan = ""; + if (globalState.getOptions().logEachSelect()) { + globalState.getLogger().writeCurrent(selectStr); + try { + globalState.getLogger().getCurrentFileWriter().flush(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + List invalidList = List.of("create", "notify", "discard", "listen", "unlisten", "reset","set","delete","alter","analyze"); + if(invalidList.contains(selectStr.split("\\s+")[0].toLowerCase())){ + return ""; + } + + SQLQueryAdapter q = new SQLQueryAdapter(PostgresExplainGenerator.explain(selectStr),null); + try (SQLancerResultSet rs = q.executeAndGet(globalState)) { + while (rs.next()) { + queryPlan += rs.getString(1); + } + } catch (SQLException | AssertionError e) { + queryPlan = ""; + } + return queryPlan; + } + + @Override + protected double[] initializeWeightedAverageReward() { + return new double[PostgresProvider.Action.values().length]; + } + + @Override + protected void executeMutator(int index, PostgresGlobalState globalState) throws Exception { + SQLQueryAdapter queryMutateTable = PostgresProvider.Action.values()[index].getQuery(globalState); + globalState.executeStatement(queryMutateTable); + } + + @Override + protected boolean addRowsToAllTables(PostgresGlobalState globalState) throws Exception { + List tablesNoRow = globalState.getSchema().getDatabaseTables().stream() + .filter(t -> t.getNrRows(globalState) == 0).collect(Collectors.toList()); + for (PostgresSchema.PostgresTable table : tablesNoRow) { + SQLQueryAdapter queryAddRows = PostgresInsertGenerator.insertRows(globalState, table); + globalState.executeStatement(queryAddRows); + } + return true; + } + } diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index 318e5eead..9d270c711 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -1,6 +1,5 @@ package sqlancer.postgres.gen; -import jnr.ffi.Struct; import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 41c017b2e..71dc9a852 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -7,6 +7,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; @@ -19,6 +20,9 @@ private PostgresInsertGenerator() { public static SQLQueryAdapter insert(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); + return insertRows(globalState,table); + } + public static SQLQueryAdapter insertRows(PostgresGlobalState globalState, PostgresSchema.PostgresTable table) { ExpectedErrors errors = new ExpectedErrors(); errors.add("cannot insert into column"); PostgresCommon.addCommonExpressionErrors(errors); From b4e619c5c051bb0ff97d429bd519f4acf3cdfd6d Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 5 Mar 2025 11:18:01 +0200 Subject: [PATCH 623/856] remove code not related to QPG support --- src/sqlancer/postgres/PostgresProvider.java | 2 -- .../gen/PostgresExplainGenerator.java | 21 +------------------ 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index f88c2c65b..eb406a505 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -57,7 +57,6 @@ protected PostgresProvider(Class globalClass, Class { - EXPLAIN(PostgresExplainGenerator::explain), ANALYZE(PostgresAnalyzeGenerator::create), // ALTER_TABLE(g -> PostgresAlterTableGenerator.create(g.getSchema().getRandomTable(t -> !t.isView()), g, generateOnlyKnown)), // @@ -131,7 +130,6 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { nrPerformed = r.getInteger(0, 5); break; case COMMIT: - case EXPLAIN: nrPerformed = r.getInteger(0, 0); break; case ALTER_TABLE: diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index 9d270c711..db082ff68 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -1,30 +1,11 @@ package sqlancer.postgres.gen; -import sqlancer.Randomly; -import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.PostgresProvider; -import sqlancer.postgres.PostgresProvider.Action; - public class PostgresExplainGenerator { private PostgresExplainGenerator(){ } - - public static SQLQueryAdapter explain(PostgresGlobalState globalState) throws Exception { - StringBuilder sb = new StringBuilder(); - sb.append("EXPLAIN "); - Action action; - - do { - action = Randomly.fromOptions(PostgresProvider.Action.values()); - } while (action == Action.EXPLAIN); - SQLQueryAdapter query = action.getQuery(globalState); - sb.append(query); - return new SQLQueryAdapter(sb.toString(),query.getExpectedErrors()); - } - + public static String explain(String selectStr) throws Exception{ StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN "); From d3d02ff03658515305efed63cbc0f85e2eb9834a Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 5 Mar 2025 12:15:35 +0200 Subject: [PATCH 624/856] format query plan string --- src/sqlancer/postgres/PostgresProvider.java | 25 ++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index eb406a505..fc38b45ff 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -360,7 +360,7 @@ public String getQueryPlan(String selectStr, PostgresGlobalState globalState) th } catch (SQLException | AssertionError e) { queryPlan = ""; } - return queryPlan; + return formatQueryPlan(queryPlan); } @Override @@ -385,4 +385,27 @@ protected boolean addRowsToAllTables(PostgresGlobalState globalState) throws Exc return true; } + private String formatQueryPlan(String queryPlan){ + StringBuilder outQueryPlanFormatted = new StringBuilder(); + boolean insideBrackets = false; + + for(char ch: queryPlan.toCharArray()){ + if(ch == '\n' || ch == ' '){ + continue; + } + if(ch == '('){ + insideBrackets = true; + } + else if(ch == ')'){ + insideBrackets = false; + outQueryPlanFormatted.append(';'); + continue; + } + if (!insideBrackets) + outQueryPlanFormatted.append(ch); + } + + return outQueryPlanFormatted.toString(); + } + } From 9c5509f4144951e0eb6d9dc5367dda90374f1630 Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 5 Mar 2025 12:17:00 +0200 Subject: [PATCH 625/856] add integration test for QPG postgres --- .../qpg/postgres/TestPostgresQPG.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test/sqlancer/qpg/postgres/TestPostgresQPG.java diff --git a/test/sqlancer/qpg/postgres/TestPostgresQPG.java b/test/sqlancer/qpg/postgres/TestPostgresQPG.java new file mode 100644 index 000000000..03e7b7ab1 --- /dev/null +++ b/test/sqlancer/qpg/postgres/TestPostgresQPG.java @@ -0,0 +1,21 @@ +package sqlancer.qpg.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.dbms.TestConfig; + +public class TestPostgresQPG { + + @Test + public void testPostgresQPG() { + String postgres = System.getenv("POSTGRES_AVAILABLE"); + boolean postgresIsAvailable = postgres != null && postgres.equalsIgnoreCase("true"); + assumeTrue(postgresIsAvailable); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "--username", + "postgres", "postgres", "--oracle", "NOREC"})); + } +} From 645b92a6d9e134ce00e71a2a4f62fdb6f20ad960 Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 5 Mar 2025 12:17:44 +0200 Subject: [PATCH 626/856] add unit test for postgres::getQueryPlan --- .../qpg/postgres/TestPostgresQueryPlan.java | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java diff --git a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java new file mode 100644 index 000000000..5901f84b7 --- /dev/null +++ b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java @@ -0,0 +1,41 @@ +package sqlancer.qpg.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.Test; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; +import sqlancer.postgres.PostgresProvider; + +public class TestPostgresQueryPlan { + + + @Test + void testPostgresQueryPlan() throws Exception { + String databaseName = "postgres"; + PostgresProvider provider = new PostgresProvider(); + PostgresGlobalState state = provider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + PostgresOptions postgresOption = provider.getOptionClass().getDeclaredConstructor().newInstance(); + state.setDbmsSpecificOptions(postgresOption); + state.setDatabaseName(databaseName); + MainOptions options = new MainOptions(); + state.setMainOptions(options); + state.setState(provider.getStateToReproduce(databaseName)); + SQLConnection con = provider.createDatabase(state); + state.setConnection(con); + Main.StateLogger logger = new Main.StateLogger(databaseName, provider, options); + state.setStateLogger(logger); + + SQLQueryAdapter q = new SQLQueryAdapter("CREATE TABLE t1(a INT, b INT);", true); + q.execute(state); + q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); + q.execute(state); + String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); + assertEquals("NestedLoopLeftJoin;->SeqScanont2;->Materialize;->SeqScanont1;Filter:;", queryPlan); + } + +} From 502a69ba10a55efaf4809b3287557d796a52f7e2 Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Fri, 7 Mar 2025 02:54:49 +0200 Subject: [PATCH 627/856] fix checkstyle violations --- src/sqlancer/postgres/PostgresProvider.java | 44 ++++++++++++++----- .../gen/PostgresExplainGenerator.java | 8 ++-- .../postgres/gen/PostgresInsertGenerator.java | 3 +- .../qpg/postgres/TestPostgresQPG.java | 7 +-- .../qpg/postgres/TestPostgresQueryPlan.java | 1 - 5 files changed, 42 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index fc38b45ff..32f74789c 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -25,7 +25,27 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryProvider; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.postgres.gen.*; +import sqlancer.postgres.gen.PostgresAlterTableGenerator; +import sqlancer.postgres.gen.PostgresAnalyzeGenerator; +import sqlancer.postgres.gen.PostgresClusterGenerator; +import sqlancer.postgres.gen.PostgresCommentGenerator; +import sqlancer.postgres.gen.PostgresDeleteGenerator; +import sqlancer.postgres.gen.PostgresDiscardGenerator; +import sqlancer.postgres.gen.PostgresDropIndexGenerator; +import sqlancer.postgres.gen.PostgresExplainGenerator; +import sqlancer.postgres.gen.PostgresIndexGenerator; +import sqlancer.postgres.gen.PostgresInsertGenerator; +import sqlancer.postgres.gen.PostgresNotifyGenerator; +import sqlancer.postgres.gen.PostgresReindexGenerator; +import sqlancer.postgres.gen.PostgresSequenceGenerator; +import sqlancer.postgres.gen.PostgresSetGenerator; +import sqlancer.postgres.gen.PostgresStatisticsGenerator; +import sqlancer.postgres.gen.PostgresTableGenerator; +import sqlancer.postgres.gen.PostgresTransactionGenerator; +import sqlancer.postgres.gen.PostgresTruncateGenerator; +import sqlancer.postgres.gen.PostgresUpdateGenerator; +import sqlancer.postgres.gen.PostgresVacuumGenerator; +import sqlancer.postgres.gen.PostgresViewGenerator; // EXISTS // IN @@ -334,7 +354,6 @@ public String getDBMSName() { return "postgres"; } - @Override public String getQueryPlan(String selectStr, PostgresGlobalState globalState) throws Exception { String queryPlan = ""; @@ -347,12 +366,13 @@ public String getQueryPlan(String selectStr, PostgresGlobalState globalState) th } } - List invalidList = List.of("create", "notify", "discard", "listen", "unlisten", "reset","set","delete","alter","analyze"); - if(invalidList.contains(selectStr.split("\\s+")[0].toLowerCase())){ + List invalidList = List.of("create", "notify", "discard", "listen", "unlisten", "reset", "set", + "delete", "alter", "analyze"); + if (invalidList.contains(selectStr.split("\\s+")[0].toLowerCase())) { return ""; } - SQLQueryAdapter q = new SQLQueryAdapter(PostgresExplainGenerator.explain(selectStr),null); + SQLQueryAdapter q = new SQLQueryAdapter(PostgresExplainGenerator.explain(selectStr), null); try (SQLancerResultSet rs = q.executeAndGet(globalState)) { while (rs.next()) { queryPlan += rs.getString(1); @@ -385,24 +405,24 @@ protected boolean addRowsToAllTables(PostgresGlobalState globalState) throws Exc return true; } - private String formatQueryPlan(String queryPlan){ + private String formatQueryPlan(String queryPlan) { StringBuilder outQueryPlanFormatted = new StringBuilder(); boolean insideBrackets = false; - for(char ch: queryPlan.toCharArray()){ - if(ch == '\n' || ch == ' '){ + for (char ch : queryPlan.toCharArray()) { + if (ch == '\n' || ch == ' ') { continue; } - if(ch == '('){ + if (ch == '(') { insideBrackets = true; - } - else if(ch == ')'){ + } else if (ch == ')') { insideBrackets = false; outQueryPlanFormatted.append(';'); continue; } - if (!insideBrackets) + if (!insideBrackets) { outQueryPlanFormatted.append(ch); + } } return outQueryPlanFormatted.toString(); diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index db082ff68..d55d4b72c 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -1,12 +1,12 @@ package sqlancer.postgres.gen; -public class PostgresExplainGenerator { +public final class PostgresExplainGenerator { - private PostgresExplainGenerator(){ + private PostgresExplainGenerator() { } - - public static String explain(String selectStr) throws Exception{ + + public static String explain(String selectStr) throws Exception { StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN "); sb.append(selectStr); diff --git a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java index 71dc9a852..49b94b184 100644 --- a/src/sqlancer/postgres/gen/PostgresInsertGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresInsertGenerator.java @@ -20,8 +20,9 @@ private PostgresInsertGenerator() { public static SQLQueryAdapter insert(PostgresGlobalState globalState) { PostgresTable table = globalState.getSchema().getRandomTable(t -> t.isInsertable()); - return insertRows(globalState,table); + return insertRows(globalState, table); } + public static SQLQueryAdapter insertRows(PostgresGlobalState globalState, PostgresSchema.PostgresTable table) { ExpectedErrors errors = new ExpectedErrors(); errors.add("cannot insert into column"); diff --git a/test/sqlancer/qpg/postgres/TestPostgresQPG.java b/test/sqlancer/qpg/postgres/TestPostgresQPG.java index 03e7b7ab1..20ec3e91e 100644 --- a/test/sqlancer/qpg/postgres/TestPostgresQPG.java +++ b/test/sqlancer/qpg/postgres/TestPostgresQPG.java @@ -14,8 +14,9 @@ public void testPostgresQPG() { String postgres = System.getenv("POSTGRES_AVAILABLE"); boolean postgresIsAvailable = postgres != null && postgres.equalsIgnoreCase("true"); assumeTrue(postgresIsAvailable); - assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, "--username", - "postgres", "postgres", "--oracle", "NOREC"})); + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "4", "--qpg-enable", "true", "--num-queries", TestConfig.NUM_QUERIES, + "--username", "postgres", "postgres", "--oracle", "NOREC" })); } } diff --git a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java index 5901f84b7..f6a71df8f 100644 --- a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java +++ b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java @@ -13,7 +13,6 @@ public class TestPostgresQueryPlan { - @Test void testPostgresQueryPlan() throws Exception { String databaseName = "postgres"; From 3c9bcb49d16a3ee54ed79fa5f9301e47782e6b92 Mon Sep 17 00:00:00 2001 From: SharmaNishchay <146124877+SharmaNishchay@users.noreply.github.com> Date: Sun, 9 Mar 2025 17:40:24 +0530 Subject: [PATCH 628/856] Refactor: Replace .size() == 0 or .length() > 0 etc checks with .isEmpty() --- .../clickhouse/ast/constant/ClickHouseStringConstant.java | 2 +- src/sqlancer/cnosdb/CnosDBLoggableFactory.java | 2 +- src/sqlancer/cnosdb/CnosDBToStringVisitor.java | 2 +- .../cockroachdb/gen/CockroachDBExpressionGenerator.java | 6 +++--- .../cockroachdb/oracle/CockroachDBCERTOracle.java | 4 ++-- .../cockroachdb/oracle/tlp/CockroachDBTLPBase.java | 2 +- src/sqlancer/common/log/SQLLoggableFactory.java | 2 +- src/sqlancer/doris/gen/DorisDropViewGenerator.java | 2 +- src/sqlancer/mariadb/ast/MariaDBStringVisitor.java | 2 +- src/sqlancer/materialize/MaterializeToStringVisitor.java | 2 +- src/sqlancer/mysql/MySQLToStringVisitor.java | 4 ++-- src/sqlancer/mysql/ast/MySQLBinaryOperation.java | 8 ++++---- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 4 ++-- src/sqlancer/oceanbase/OceanBaseToStringVisitor.java | 6 +++--- src/sqlancer/oceanbase/ast/OceanBaseConstant.java | 2 +- src/sqlancer/postgres/PostgresToStringVisitor.java | 2 +- .../postgres/gen/PostgresExpressionGenerator.java | 4 ++-- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 2 +- src/sqlancer/tidb/TiDBExpressionGenerator.java | 4 ++-- src/sqlancer/tidb/gen/TiDBDropViewGenerator.java | 2 +- src/sqlancer/tidb/gen/TiDBTableGenerator.java | 2 +- src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java | 2 +- 22 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java index bde359f92..fa8ba11f5 100644 --- a/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java +++ b/src/sqlancer/clickhouse/ast/constant/ClickHouseStringConstant.java @@ -32,7 +32,7 @@ public String toString() { @Override public boolean asBooleanNotNull() { - return value.length() > 0; + return !value.isEmpty(); } @Override diff --git a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java index a3f361b14..407621c8b 100644 --- a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java +++ b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java @@ -19,7 +19,7 @@ protected Loggable createLoggable(String input, String suffix) { if (!input.endsWith(";")) { completeString += ";"; } - if (suffix != null && suffix.length() != 0) { + if (suffix != null && !suffix.isEmpty()) { completeString += suffix; } return new LoggedString(completeString); diff --git a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java index 44d3bf509..388e2ccd8 100644 --- a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java +++ b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java @@ -137,7 +137,7 @@ public void visit(CnosDBSelect s) { sb.append(" WHERE "); visit(s.getWhereClause()); } - if (s.getGroupByExpressions().size() > 0) { + if (!s.getGroupByExpressions().isEmpty()) { sb.append(" GROUP BY "); visit(s.getGroupByExpressions()); } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java index 52d8f027d..8aeb5492b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBExpressionGenerator.java @@ -457,7 +457,7 @@ public String generateUnoptimizedQueryString(CockroachDBSelect select, Cockroach @Override public List generateFetchColumns(boolean shouldCreateDummy) { - if (shouldCreateDummy || columns.size() == 0) { + if (shouldCreateDummy || columns.isEmpty()) { return List.of(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); } return Randomly.nonEmptySubset(columns).stream().map(c -> new CockroachDBColumnReference(c)) @@ -541,7 +541,7 @@ boolean mutateWhere(CockroachDBSelect select) { } boolean mutateGroupBy(CockroachDBSelect select) { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); } else { @@ -551,7 +551,7 @@ boolean mutateGroupBy(CockroachDBSelect select) { } boolean mutateHaving(CockroachDBSelect select) { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(generateExpression(CockroachDBDataType.BOOL.get())); return false; diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java index 07a51861f..7d2af5930 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCERTOracle.java @@ -177,7 +177,7 @@ protected boolean mutateWhere() { @Override protected boolean mutateGroupBy() { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); } else { @@ -188,7 +188,7 @@ protected boolean mutateGroupBy() { @Override protected boolean mutateHaving() { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(gen.generateExpression(CockroachDBDataType.BOOL.get())); return false; diff --git a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java index c4809d231..d84cab56f 100644 --- a/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java +++ b/src/sqlancer/cockroachdb/oracle/tlp/CockroachDBTLPBase.java @@ -56,7 +56,7 @@ public void check() throws SQLException { List generateFetchColumns() { List columns = new ArrayList<>(); - if (Randomly.getBoolean() || targetTables.getColumns().size() == 0) { + if (Randomly.getBoolean() || targetTables.getColumns().isEmpty()) { columns.add(new CockroachDBColumnReference(new CockroachDBColumn("*", null, false, false))); } else { columns.addAll(Randomly.nonEmptySubset(targetTables.getColumns()).stream() diff --git a/src/sqlancer/common/log/SQLLoggableFactory.java b/src/sqlancer/common/log/SQLLoggableFactory.java index 7b7dfe4f1..bdcf2253f 100644 --- a/src/sqlancer/common/log/SQLLoggableFactory.java +++ b/src/sqlancer/common/log/SQLLoggableFactory.java @@ -16,7 +16,7 @@ protected Loggable createLoggable(String input, String suffix) { } completeString = completeString.replace("\n", "\\n"); completeString = completeString.replace("\r", "\\r"); - if (suffix != null && suffix.length() != 0) { + if (suffix != null && !suffix.isEmpty()) { completeString += suffix; } return new LoggedString(completeString); diff --git a/src/sqlancer/doris/gen/DorisDropViewGenerator.java b/src/sqlancer/doris/gen/DorisDropViewGenerator.java index 81c1629c2..4f87ba88d 100644 --- a/src/sqlancer/doris/gen/DorisDropViewGenerator.java +++ b/src/sqlancer/doris/gen/DorisDropViewGenerator.java @@ -11,7 +11,7 @@ private DorisDropViewGenerator() { } public static SQLQueryAdapter dropView(DorisGlobalState globalState) { - if (globalState.getSchema().getTables(t -> t.isView()).size() == 0) { + if (globalState.getSchema().getTables(t -> t.isView()).isEmpty()) { throw new IgnoreMeException(); } StringBuilder sb = new StringBuilder("DROP VIEW "); diff --git a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java index e1dbf476f..f7fed0c83 100644 --- a/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java +++ b/src/sqlancer/mariadb/ast/MariaDBStringVisitor.java @@ -54,7 +54,7 @@ public void visit(MariaDBSelectStatement s) { sb.append(" WHERE "); visit(s.getWhereCondition()); } - if (s.getGroupBys().size() != 0) { + if (!s.getGroupBys().isEmpty()) { sb.append(" GROUP BY "); for (i = 0; i < s.getGroupBys().size(); i++) { if (i != 0) { diff --git a/src/sqlancer/materialize/MaterializeToStringVisitor.java b/src/sqlancer/materialize/MaterializeToStringVisitor.java index b39179cae..285be276f 100644 --- a/src/sqlancer/materialize/MaterializeToStringVisitor.java +++ b/src/sqlancer/materialize/MaterializeToStringVisitor.java @@ -139,7 +139,7 @@ public void visit(MaterializeSelect s) { sb.append(" WHERE "); visit(s.getWhereClause()); } - if (s.getGroupByExpressions().size() > 0) { + if (!s.getGroupByExpressions().isEmpty()) { sb.append(" GROUP BY "); visit(s.getGroupByExpressions()); } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index 4d9607717..b459fe81e 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -59,7 +59,7 @@ public void visit(MySQLSelect s) { throw new AssertionError(); } sb.append(s.getModifiers().stream().collect(Collectors.joining(" "))); - if (s.getModifiers().size() > 0) { + if (!s.getModifiers().isEmpty()) { sb.append(" "); } if (s.getFetchColumns() == null) { @@ -92,7 +92,7 @@ public void visit(MySQLSelect s) { sb.append(" WHERE "); visit(whereClause); } - if (s.getGroupByExpressions() != null && s.getGroupByExpressions().size() > 0) { + if (s.getGroupByExpressions() != null && !s.getGroupByExpressions().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); List groupBys = s.getGroupByExpressions(); diff --git a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java index 33293f26b..b3e0714cb 100644 --- a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java +++ b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java @@ -78,20 +78,20 @@ public MySQLConstant getExpectedValue() { /* workaround for https://bugs.mysql.com/bug.php?id=95960 */ if (leftExpected.isString()) { String text = leftExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && text.length() > 0) { + while ((text.startsWith(" ") || text.startsWith("\t")) && !text.isEmpty()) { text = text.substring(1); } - if (text.length() > 0 && (text.startsWith("\n") || text.startsWith("."))) { + if (!text.isEmpty() && (text.startsWith("\n") || text.startsWith("."))) { throw new IgnoreMeException(); } } if (rightExpected.isString()) { String text = rightExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && text.length() > 0) { + while ((text.startsWith(" ") || text.startsWith("\t")) && !text.isEmpty()) { text = text.substring(1); } - if (text.length() > 0 && (text.startsWith("\n") || text.startsWith("."))) { + if (!text.isEmpty() && (text.startsWith("\n") || text.startsWith("."))) { throw new IgnoreMeException(); } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index f4b868f61..f23c48c29 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -301,7 +301,7 @@ boolean mutateWhere(MySQLSelect select) { } boolean mutateGroupBy(MySQLSelect select) { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); } else { @@ -311,7 +311,7 @@ boolean mutateGroupBy(MySQLSelect select) { } boolean mutateHaving(MySQLSelect select) { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(generateExpression()); return false; diff --git a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java index a65ae8ee3..ba306ff90 100644 --- a/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java +++ b/src/sqlancer/oceanbase/OceanBaseToStringVisitor.java @@ -56,7 +56,7 @@ public void visit(OceanBaseSelect s) { throw new AssertionError(); } sb.append(s.getModifiers().stream().collect(Collectors.joining(" "))); - if (s.getModifiers().size() > 0) { + if (!s.getModifiers().isEmpty()) { sb.append(" "); } if (s.getFetchColumns() == null) { @@ -85,7 +85,7 @@ public void visit(OceanBaseSelect s) { sb.append(" WHERE "); visit(whereClause); } - if (s.getGroupByExpressions() != null && s.getGroupByExpressions().size() > 0) { + if (s.getGroupByExpressions() != null && !s.getGroupByExpressions().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); List groupBys = s.getGroupByExpressions(); @@ -260,7 +260,7 @@ public void visit(OceanBaseStringExpression op) { sb.append(op.getStr()); } else { String str = op.getStr(); - if (str.length() > 0) { + if (!str.isEmpty()) { sb.append(r.getInteger(0, 100000)); } else { sb.append(r.getInteger(0, 1000000)); diff --git a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java index 91a481fc6..84dadb9e4 100644 --- a/src/sqlancer/oceanbase/ast/OceanBaseConstant.java +++ b/src/sqlancer/oceanbase/ast/OceanBaseConstant.java @@ -194,7 +194,7 @@ public boolean isNull() { @Override public boolean isEmpty() { // "" " " - if (value.length() == 0) { + if (value.isEmpty()) { return true; } else { for (int i = 0; i < value.length(); i++) { diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 17327ab9c..78475a0af 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -161,7 +161,7 @@ public void visit(PostgresSelect s) { sb.append(" WHERE "); visit(s.getWhereClause()); } - if (s.getGroupByExpressions().size() > 0) { + if (!s.getGroupByExpressions().isEmpty()) { sb.append(" GROUP BY "); visit(s.getGroupByExpressions()); } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 5f081ef2a..c6189acc5 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -819,7 +819,7 @@ boolean mutateWhere(PostgresSelect select) { } boolean mutateGroupBy(PostgresSelect select) { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); } else { @@ -829,7 +829,7 @@ boolean mutateGroupBy(PostgresSelect select) { } boolean mutateHaving(PostgresSelect select) { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(generateExpression(0, PostgresDataType.BOOLEAN)); return false; diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index b8b860943..09eb224fb 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -139,7 +139,7 @@ public void visit(SQLite3Select s, boolean inner) { visit(whereClause); sb.append(")"); } - if (s.getGroupByClause().size() > 0) { + if (!s.getGroupByClause().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); visit(s.getGroupByClause()); diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index ff6a529d0..8eaca35f0 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -318,7 +318,7 @@ boolean mutateWhere(TiDBSelect select) { } boolean mutateHaving(TiDBSelect select) { - if (select.getGroupByExpressions().size() == 0) { + if (select.getGroupByExpressions().isEmpty()) { select.setGroupByExpressions(select.getFetchColumns()); select.setHavingClause(generateExpression()); return false; @@ -367,7 +367,7 @@ boolean mutateLimit(TiDBSelect select) { } private boolean mutateGroupBy(TiDBSelect select) { - boolean increase = select.getGroupByExpressions().size() > 0; + boolean increase = !select.getGroupByExpressions().isEmpty(); if (increase) { select.clearGroupByExpressions(); select.clearHavingClause(); diff --git a/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java index 48bfc8a46..486b5f873 100644 --- a/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDropViewGenerator.java @@ -11,7 +11,7 @@ private TiDBDropViewGenerator() { } public static SQLQueryAdapter dropView(TiDBGlobalState globalState) { - if (globalState.getSchema().getTables(t -> t.isView()).size() == 0) { + if (globalState.getSchema().getTables(t -> t.isView()).isEmpty()) { throw new IgnoreMeException(); } StringBuilder sb = new StringBuilder("DROP VIEW "); diff --git a/src/sqlancer/tidb/gen/TiDBTableGenerator.java b/src/sqlancer/tidb/gen/TiDBTableGenerator.java index af9ad4457..451681409 100644 --- a/src/sqlancer/tidb/gen/TiDBTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBTableGenerator.java @@ -47,7 +47,7 @@ public SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException StringBuilder sb = new StringBuilder("CREATE TABLE "); sb.append(tableName); - if (Randomly.getBoolean() && globalState.getSchema().getDatabaseTables().size() > 0) { + if (Randomly.getBoolean() && !globalState.getSchema().getDatabaseTables().isEmpty()) { sb.append(" LIKE "); TiDBTable otherTable = globalState.getSchema().getRandomTable(); sb.append(otherTable.getName()); diff --git a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java index eef4c7d9f..54e718768 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java +++ b/src/sqlancer/yugabyte/ysql/YSQLToStringVisitor.java @@ -127,7 +127,7 @@ public void visit(YSQLSelect s) { sb.append(" WHERE "); visit(s.getWhereClause()); } - if (s.getGroupByExpressions().size() > 0) { + if (!s.getGroupByExpressions().isEmpty()) { sb.append(" GROUP BY "); visit(s.getGroupByExpressions()); } From 37e72a04c578d5ee55efb7bfac013e28264b8fe3 Mon Sep 17 00:00:00 2001 From: Nishchay Sharma <146124877+SharmaNishchay@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:22:46 +0530 Subject: [PATCH 629/856] Fixes the typo on line 598 (#1097) --- src/sqlancer/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 46841095b..7df750bf1 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -595,7 +595,7 @@ public void run() { System.out.println( formatInteger(nrSuccessfulActions.get()) + " successfully-executed statements"); System.out.println( - formatInteger(nrUnsuccessfulActions.get()) + " unsuccessfuly-executed statements"); + formatInteger(nrUnsuccessfulActions.get()) + " unsuccessfully-executed statements"); } private String formatInteger(long intValue) { From 52d7224f13ed9be81c8befb1f0dd19b620e3122c Mon Sep 17 00:00:00 2001 From: SharmaNishchay <146124877+SharmaNishchay@users.noreply.github.com> Date: Tue, 11 Mar 2025 21:51:14 +0530 Subject: [PATCH 630/856] Remove redundant text.length() checks --- src/sqlancer/mysql/ast/MySQLBinaryOperation.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java index b3e0714cb..af0b0ccf7 100644 --- a/src/sqlancer/mysql/ast/MySQLBinaryOperation.java +++ b/src/sqlancer/mysql/ast/MySQLBinaryOperation.java @@ -78,20 +78,20 @@ public MySQLConstant getExpectedValue() { /* workaround for https://bugs.mysql.com/bug.php?id=95960 */ if (leftExpected.isString()) { String text = leftExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && !text.isEmpty()) { + while (text.startsWith(" ") || text.startsWith("\t")) { text = text.substring(1); } - if (!text.isEmpty() && (text.startsWith("\n") || text.startsWith("."))) { + if (text.startsWith("\n") || text.startsWith(".")) { throw new IgnoreMeException(); } } if (rightExpected.isString()) { String text = rightExpected.castAsString(); - while ((text.startsWith(" ") || text.startsWith("\t")) && !text.isEmpty()) { + while (text.startsWith(" ") || text.startsWith("\t")) { text = text.substring(1); } - if (!text.isEmpty() && (text.startsWith("\n") || text.startsWith("."))) { + if (text.startsWith("\n") || text.startsWith(".")) { throw new IgnoreMeException(); } } From 049176b73af1e0f5af7c24fb2e581b9a265424b1 Mon Sep 17 00:00:00 2001 From: Amr Riyad Date: Tue, 11 Mar 2025 20:43:40 +0200 Subject: [PATCH 631/856] [CockroachDB] Add 'excluded' support for 'ON CONFLICT' clause --- .../cockroachdb/gen/CockroachDBInsertGenerator.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java index decbe927f..9f823c59b 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBInsertGenerator.java @@ -81,8 +81,6 @@ public static SQLQueryAdapter insert(CockroachDBGlobalState globalState, Cockroa if (Randomly.getBoolean()) { sb.append(" NOTHING "); } else { - // TODO: also support excluded. (see - // https://www.cockroachlabs.com/docs/stable/insert.html) sb.append(" UPDATE SET "); List columns = table.getRandomNonEmptyColumnSubset(); int i = 0; @@ -92,7 +90,12 @@ public static SQLQueryAdapter insert(CockroachDBGlobalState globalState, Cockroa } sb.append(c.getName()); sb.append(" = "); - sb.append(CockroachDBVisitor.asString(gen.generateConstant(c.getType()))); + if (Randomly.getBoolean()) { + sb.append(CockroachDBVisitor.asString(gen.generateConstant(c.getType()))); + } else { + sb.append("excluded."); + sb.append(c.getName()); + } } errors.add("UPSERT or INSERT...ON CONFLICT command cannot affect row a second time"); } From a2757eebc61aee4bf6d96d841ba8f80470978d7b Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Tue, 11 Mar 2025 23:34:18 +0200 Subject: [PATCH 632/856] remove unnecessary line --- src/sqlancer/postgres/PostgresProvider.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 32f74789c..8d76867c8 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -187,7 +187,6 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; - default: throw new AssertionError(a); } From 8604997fdfed123bf3dbf8afb662cfcfdb23b64e Mon Sep 17 00:00:00 2001 From: ahmed -k Date: Wed, 12 Mar 2025 02:11:23 +0200 Subject: [PATCH 633/856] extract node type using BFS --- src/sqlancer/postgres/PostgresProvider.java | 50 +++---- .../gen/PostgresExplainGenerator.java | 2 +- .../qpg/postgres/TestPostgresQueryPlan.java | 127 +++++++++++++++++- 3 files changed, 153 insertions(+), 26 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 8d76867c8..78d404b47 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -7,10 +7,15 @@ import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; +import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedList; import java.util.List; +import java.util.Queue; import java.util.stream.Collectors; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.auto.service.AutoService; import sqlancer.AbstractAction; @@ -364,13 +369,6 @@ public String getQueryPlan(String selectStr, PostgresGlobalState globalState) th e.printStackTrace(); } } - - List invalidList = List.of("create", "notify", "discard", "listen", "unlisten", "reset", "set", - "delete", "alter", "analyze"); - if (invalidList.contains(selectStr.split("\\s+")[0].toLowerCase())) { - return ""; - } - SQLQueryAdapter q = new SQLQueryAdapter(PostgresExplainGenerator.explain(selectStr), null); try (SQLancerResultSet rs = q.executeAndGet(globalState)) { while (rs.next()) { @@ -404,27 +402,31 @@ protected boolean addRowsToAllTables(PostgresGlobalState globalState) throws Exc return true; } - private String formatQueryPlan(String queryPlan) { - StringBuilder outQueryPlanFormatted = new StringBuilder(); - boolean insideBrackets = false; + public String formatQueryPlan(String queryPlan) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JsonNode root = mapper.readTree(queryPlan).get(0).get("Plan"); + // Extract nodes using BFS algorithm + List nodeTypes = extractNodeTypesIterative(root); + return String.join(" ", nodeTypes); + } - for (char ch : queryPlan.toCharArray()) { - if (ch == '\n' || ch == ' ') { - continue; + // BFS algorithm for traversing the Json Query Plan + private static List extractNodeTypesIterative(JsonNode root) { + List result = new ArrayList<>(); + Queue queue = new LinkedList<>(); + queue.add(root); + while (!queue.isEmpty()) { + JsonNode node = queue.poll(); + if (node.has("Node Type")) { + result.add(node.get("Node Type").asText()); } - if (ch == '(') { - insideBrackets = true; - } else if (ch == ')') { - insideBrackets = false; - outQueryPlanFormatted.append(';'); - continue; - } - if (!insideBrackets) { - outQueryPlanFormatted.append(ch); + if (node.has("Plans") && node.get("Plans").isArray()) { + for (JsonNode plan : node.get("Plans")) { + queue.add(plan); + } } } - - return outQueryPlanFormatted.toString(); + return result; } } diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index d55d4b72c..d3039394b 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -8,7 +8,7 @@ private PostgresExplainGenerator() { public static String explain(String selectStr) throws Exception { StringBuilder sb = new StringBuilder(); - sb.append("EXPLAIN "); + sb.append("EXPLAIN (FORMAT JSON) "); sb.append(selectStr); return sb.toString(); } diff --git a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java index f6a71df8f..c883e87d8 100644 --- a/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java +++ b/test/sqlancer/qpg/postgres/TestPostgresQueryPlan.java @@ -34,7 +34,132 @@ void testPostgresQueryPlan() throws Exception { q = new SQLQueryAdapter("CREATE TABLE t2(c INT);", true); q.execute(state); String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); - assertEquals("NestedLoopLeftJoin;->SeqScanont2;->Materialize;->SeqScanont1;Filter:;", queryPlan); + assertEquals("Nested Loop Seq Scan Materialize Seq Scan", queryPlan); + } + + @Test + void testFormatQueryPlan() throws Exception { + + PostgresProvider provider = new PostgresProvider(); + + String queryPlan = "[\n" + " {\n" + " \"Plan\": {\n" + " \"Node Type\": \"Aggregate\",\n" + + " \"Strategy\": \"Hashed\",\n" + " \"Partial Mode\": \"Simple\",\n" + + " \"Parallel Aware\": false,\n" + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 62998.82,\n" + " \"Total Cost\": 63009.32,\n" + + " \"Plan Rows\": 1050,\n" + " \"Plan Width\": 4,\n" + " \"Output\": [\"t1.c0\"],\n" + + " \"Group Key\": [\"t1.c0\"],\n" + " \"Planned Partitions\": 0,\n" + " \"Plans\": [\n" + + " {\n" + " \"Node Type\": \"Append\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + " \"Startup Cost\": 27150.40,\n" + + " \"Total Cost\": 62996.20,\n" + " \"Plan Rows\": 1050,\n" + + " \"Plan Width\": 4,\n" + " \"Subplans Removed\": 0,\n" + " \"Plans\": [\n" + + " {\n" + " \"Node Type\": \"Group\",\n" + + " \"Parent Relationship\": \"Member\",\n" + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + " \"Startup Cost\": 27150.40,\n" + + " \"Total Cost\": 62949.08,\n" + " \"Plan Rows\": 200,\n" + + " \"Plan Width\": 4,\n" + " \"Output\": [\"t1.c0\"],\n" + + " \"Group Key\": [\"t1.c0\"],\n" + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Gather Merge\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 27150.40,\n" + " \"Total Cost\": 62948.08,\n" + + " \"Plan Rows\": 400,\n" + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"],\n" + " \"Workers Planned\": 2,\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Group\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 26150.38,\n" + + " \"Total Cost\": 61901.89,\n" + " \"Plan Rows\": 200,\n" + + " \"Plan Width\": 4,\n" + " \"Output\": [\"t1.c0\"],\n" + + " \"Group Key\": [\"t1.c0\"],\n" + " \"Plans\": [\n" + + " {\n" + " \"Node Type\": \"Merge Join\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Join Type\": \"Inner\",\n" + + " \"Startup Cost\": 26150.38,\n" + + " \"Total Cost\": 56906.48,\n" + + " \"Plan Rows\": 1998164,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"],\n" + + " \"Inner Unique\": false,\n" + + " \"Merge Cond\": \"(t0.c0 = t1.c0)\",\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Sort\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 25970.60,\n" + + " \"Total Cost\": 26362.39,\n" + + " \"Plan Rows\": 156719,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t0.c0\"],\n" + + " \"Sort Key\": [\"t0.c0\"],\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Seq Scan\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": true,\n" + + " \"Async Capable\": false,\n" + + " \"Relation Name\": \"t0\",\n" + + " \"Schema\": \"public\",\n" + + " \"Alias\": \"t0\",\n" + + " \"Startup Cost\": 0.00,\n" + + " \"Total Cost\": 10301.95,\n" + + " \"Plan Rows\": 156719,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t0.c0\"],\n" + + " \"Filter\": \"(t0.c0 < 100)\"\n" + + " }\n" + " ]\n" + + " },\n" + " {\n" + + " \"Node Type\": \"Sort\",\n" + + " \"Parent Relationship\": \"Inner\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Startup Cost\": 179.78,\n" + + " \"Total Cost\": 186.16,\n" + + " \"Plan Rows\": 2550,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"],\n" + + " \"Sort Key\": [\"t1.c0\"],\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Seq Scan\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + + " \"Relation Name\": \"t1\",\n" + + " \"Schema\": \"public\",\n" + + " \"Alias\": \"t1\",\n" + + " \"Startup Cost\": 0.00,\n" + + " \"Total Cost\": 35.50,\n" + + " \"Plan Rows\": 2550,\n" + + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t1.c0\"]\n" + " }\n" + + " ]\n" + " }\n" + + " ]\n" + " }\n" + " ]\n" + + " }\n" + " ]\n" + " }\n" + " ]\n" + + " },\n" + " {\n" + " \"Node Type\": \"Bitmap Heap Scan\",\n" + + " \"Parent Relationship\": \"Member\",\n" + " \"Parallel Aware\": false,\n" + + " \"Async Capable\": false,\n" + " \"Relation Name\": \"t2\",\n" + + " \"Schema\": \"public\",\n" + " \"Alias\": \"t2\",\n" + + " \"Startup Cost\": 10.74,\n" + " \"Total Cost\": 31.37,\n" + + " \"Plan Rows\": 850,\n" + " \"Plan Width\": 4,\n" + + " \"Output\": [\"t2.c0\"],\n" + " \"Recheck Cond\": \"(t2.c0 < 10)\",\n" + + " \"Plans\": [\n" + " {\n" + + " \"Node Type\": \"Bitmap Index Scan\",\n" + + " \"Parent Relationship\": \"Outer\",\n" + + " \"Parallel Aware\": false,\n" + " \"Async Capable\": false,\n" + + " \"Index Name\": \"t2_pkey\",\n" + " \"Startup Cost\": 0.00,\n" + + " \"Total Cost\": 10.53,\n" + " \"Plan Rows\": 850,\n" + + " \"Plan Width\": 0,\n" + " \"Index Cond\": \"(t2.c0 < 10)\"\n" + + " }\n" + " ]\n" + " }\n" + " ]\n" + " }\n" + + " ]\n" + " },\n" + " \"Planning Time\": 1.954\n" + " }\n" + "]\n"; + + String formatedQueryPlan = provider.formatQueryPlan(queryPlan); + assertEquals( + "Aggregate Append Group Bitmap Heap Scan Gather Merge Bitmap Index Scan Group Merge Join Sort Sort Seq Scan Seq Scan", + formatedQueryPlan); } } From e68056a694a8ebc31734d8e23da14e7626158898 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Wed, 12 Mar 2025 16:24:45 -0400 Subject: [PATCH 634/856] Feat(postgres): Add Window Function Support For Query Generation Implement comprehensive window function support for PostgreSQL testing, including: - Window function expressions with partition by, order by, ow_number, rank, and frame clauses - Named window definitions via window clause - Integration with existing expression generation framework The implementation allows SQLancer to generate and test queries containing window functions, expanding test coverage for PostgreSQL's analytical capabilities. Window functions are now included in the expression generation options and can be mutated during testing like other query components. Key changes: - Add PostgresWindowFunction and supporting classes - Enhance PostgresSelect to handle window clauses - Update expression generator for window function creation - Modify visitor for proper SQL generation --- .../postgres/PostgresToStringVisitor.java | 33 ++++++ src/sqlancer/postgres/PostgresVisitor.java | 7 +- src/sqlancer/postgres/ast/PostgresSelect.java | 49 +++++++++ .../postgres/ast/PostgresWindowFunction.java | 104 ++++++++++++++++++ .../gen/PostgresExpressionGenerator.java | 102 ++++++++++++++++- .../gen/PostgresWindowFunctionGenerator.java | 96 ++++++++++++++++ 6 files changed, 388 insertions(+), 3 deletions(-) create mode 100644 src/sqlancer/postgres/ast/PostgresWindowFunction.java create mode 100644 src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 78475a0af..b746bc733 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -362,4 +362,37 @@ public void visit(PostgresLikeOperation op) { super.visit((BinaryOperation) op); } + @Override + public void visit(PostgresWindowFunction windowFunction) { + sb.append(windowFunction.getFunctionName()); + sb.append("("); + visit(windowFunction.getArguments()); + sb.append(") OVER ("); + + WindowSpecification spec = windowFunction.getWindowSpec(); + if (!spec.getPartitionBy().isEmpty()) { + sb.append("PARTITION BY "); + visit(spec.getPartitionBy()); + } + + if (!spec.getOrderBy().isEmpty()) { + if (!spec.getPartitionBy().isEmpty()) { + sb.append(" "); + } + sb.append("ORDER BY "); + visit(spec.getOrderBy()); + } + + if (spec.getFrame() != null) { + sb.append(" "); + WindowFrame frame = spec.getFrame(); + sb.append(frame.getType().getSQL()); + sb.append(" BETWEEN "); + visit(frame.getStartExpr()); + sb.append(" AND "); + visit(frame.getEndExpr()); + } + + sb.append(")"); + } } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index d9c82232c..5951b1893 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -72,6 +72,8 @@ public interface PostgresVisitor { void visit(PostgresLikeOperation op); + void visit(PostgresWindowFunction windowFunction); + default void visit(PostgresExpression expression) { if (expression instanceof PostgresConstant) { visit((PostgresConstant) expression); @@ -113,7 +115,10 @@ default void visit(PostgresExpression expression) { visit((PostgresColumnReference) expression); } else if (expression instanceof PostgresTableReference) { visit((PostgresTableReference) expression); - } else { + } else if (expression instanceof PostgresWindowFunction) { + visit((PostgresWindowFunction) expression); + } + else { throw new AssertionError(expression); } } diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index c864a56e8..4b7773214 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -37,6 +37,55 @@ public static ForClause getRandom() { } } + public static class WindowDefinition { + private final List partitionBy; + private final List orderBy; + private final WindowFrame frame; + + public WindowDefinition(List partitionBy, + List orderBy, + WindowFrame frame) { + this.partitionBy = partitionBy; + this.orderBy = orderBy; + this.frame = frame; + } + + public List getPartitionBy() { + return partitionBy; + } + + public List getOrderBy() { + return orderBy; + } + + public WindowFrame getFrame() { + return frame; + } + } + + // Getters setters for windowfunctions + public List getWindowFunctions() { + return windowFunctions; + } + + public void setWindowFunctions(List windowFunctions) { + this.windowFunctions = windowFunctions != null ? windowFunctions : new ArrayList<>(); + } + + // Add methods for window definitions + public void addWindowDefinition(String name, WindowDefinition definition) { + windowDefinitions.put(name, definition); + } + + public WindowDefinition getWindowDefinition(String name) { + return windowDefinitions.get(name); + } + + public Map getWindowDefinitions() { + return windowDefinitions; + } + + public static class PostgresFromTable implements PostgresExpression { private final PostgresTable t; private final boolean only; diff --git a/src/sqlancer/postgres/ast/PostgresWindowFunction.java b/src/sqlancer/postgres/ast/PostgresWindowFunction.java new file mode 100644 index 000000000..dd92abbae --- /dev/null +++ b/src/sqlancer/postgres/ast/PostgresWindowFunction.java @@ -0,0 +1,104 @@ +package sqlancer.postgres.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.postgres.PostgresSchema.PostgresDataType; + +public class PostgresWindowFunction implements PostgresExpression { + + private final String functionName; + private final List arguments; + private final WindowSpecification windowSpec; + private final PostgresDataType returnType; + + public static class WindowSpecification { + private final List partitionBy; + private final List orderBy; + private final WindowFrame frame; + + public WindowSpecification(List partitionBy, + List orderBy, + WindowFrame frame) { + this.partitionBy = partitionBy; + this.orderBy = orderBy; + this.frame = frame; + } + + public List getPartitionBy() { + return partitionBy; + } + + public List getOrderBy() { + return orderBy; + } + + public WindowFrame getFrame() { + return frame; + } + } + + public static class WindowFrame { + public enum FrameType { + ROWS("ROWS"), + RANGE("RANGE"); + + private final String sql; + + FrameType(String sql) { + this.sql = sql; + } + + public String getSQL() { + return sql; + } + } + + private final FrameType type; + private final PostgresExpression startExpr; + private final PostgresExpression endExpr; + + public WindowFrame(FrameType type, PostgresExpression startExpr, PostgresExpression endExpr) { + this.type = type; + this.startExpr = startExpr; + this.endExpr = endExpr; + } + + public FrameType getType() { + return type; + } + + public PostgresExpression getStartExpr() { + return startExpr; + } + + public PostgresExpression getEndExpr() { + return endExpr; + } + } + + public PostgresWindowFunction(String functionName, List arguments, + WindowSpecification windowSpec, PostgresDataType returnType) { + this.functionName = functionName; + this.arguments = arguments; + this.windowSpec = windowSpec; + this.returnType = returnType; + } + + public String getFunctionName() { + return functionName; + } + + public List getArguments() { + return arguments; + } + + public WindowSpecification getWindowSpec() { + return windowSpec; + } + + @Override + public PostgresDataType getExpressionType() { + return returnType; + } +} \ No newline at end of file diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index c6189acc5..29b02e484 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -422,6 +422,53 @@ private PostgresExpression generateTextExpression(int depth) { throw new AssertionError(); } } + + private PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { + // Generate window function arguments + List arguments = new ArrayList<>(); + if (Randomly.getBoolean()) { + arguments.add(generateExpression(depth + 1)); + } + + // Generate PARTITION BY expressions + List partitionBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + partitionBy.add(generateExpression(depth + 1)); + } + } + + // Generate ORDER BY expressions + List orderBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + PostgresExpression expr = generateExpression(depth + 1); + orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + } + } + + // Generate window frame + WindowFrame frame = null; + if (Randomly.getBoolean()) { + WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); + PostgresExpression startExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + PostgresExpression endExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + frame = new WindowFrame(frameType, startExpr, endExpr); + } + + WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + + // Select a window function + String functionName = Randomly.fromList(Arrays.asList( + "row_number", "rank", "dense_rank", "percent_rank", + "cume_dist", "ntile", "lag", "lead", "first_value", + "last_value", "nth_value" + )); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } private PostgresExpression generateConcat(int depth) { PostgresExpression left = generateExpression(depth + 1, PostgresDataType.TEXT); @@ -447,7 +494,7 @@ private PostgresExpression generateBitExpression(int depth) { } private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION } private PostgresExpression generateIntExpression(int depth) { @@ -667,7 +714,21 @@ public PostgresExpression generateBooleanExpression() { @Override public PostgresSelect generateSelect() { - return new PostgresSelect(); + PostgresSelect select = new PostgresSelect(); + + // Add window functions to fetch columns if appropriate + if (Randomly.getBoolean()) { + List windowFunctions = new ArrayList<>(); + int numWindowFunctions = Randomly.smallNumber(); + for (int i = 0; i < numWindowFunctions; i++) { + windowFunctions.add(generateWindowFunction(0, Randomly.fromList(Arrays.asList( + PostgresDataType.INT, PostgresDataType.FLOAT + )))); + } + select.setWindowFunctions(windowFunctions); + } + + return select; } @Override @@ -768,6 +829,43 @@ public boolean mutate(PostgresSelect select) { return Randomly.fromList(mutators).apply(select); } + + @Override + public boolean mutate(PostgresSelect select) { + List> mutators = new ArrayList<>(); + + mutators.add(this::mutateJoin); + mutators.add(this::mutateWhere); + mutators.add(this::mutateGroupBy); + mutators.add(this::mutateHaving); + mutators.add(this::mutateWindowFunction); // Add window function mutation + if (!PostgresBugs.bug18643) { + mutators.add(this::mutateAnd); + mutators.add(this::mutateOr); + } + mutators.add(this::mutateDistinct); + + return Randomly.fromList(mutators).apply(select); + } + + private boolean mutateWindowFunction(PostgresSelect select) { + List windowFunctions = select.getWindowFunctions(); + if (windowFunctions == null || windowFunctions.isEmpty()) { + // Add a new window function + windowFunctions = new ArrayList<>(); + windowFunctions.add(generateWindowFunction(0, PostgresDataType.INT)); + select.setWindowFunctions(windowFunctions); + return false; + } else { + // Remove a random window function + windowFunctions.remove(Randomly.fromList(windowFunctions)); + if (windowFunctions.isEmpty()) { + select.setWindowFunctions(null); + } + return true; + } + } + boolean mutateJoin(PostgresSelect select) { if (select.getJoinList().isEmpty()) { return false; diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java new file mode 100644 index 000000000..7537db9cb --- /dev/null +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -0,0 +1,96 @@ +package sqlancer.postgres.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.ast.PostgresExpression; +import sqlancer.postgres.ast.PostgresOrderByTerm; +import sqlancer.postgres.ast.PostgresWindowFunction; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; + +public class PostgresWindowFunctionGenerator { + + private static final List WINDOW_FUNCTIONS = Arrays.asList( + "row_number", "rank", "dense_rank", "percent_rank", + "cume_dist", "ntile", "lag", "lead", "first_value", + "last_value", "nth_value" + ); + + public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState globalState, + List availableExpr) { + + String functionName = Randomly.fromList(WINDOW_FUNCTIONS); + List arguments = new ArrayList<>(); + + // Generate function arguments based on function name + switch (functionName) { + case "ntile": + arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); + break; + case "lag": + case "lead": + case "nth_value": + arguments.add(Randomly.fromList(availableExpr)); + if (Randomly.getBoolean()) { + arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); + } + break; + case "first_value": + case "last_value": + arguments.add(Randomly.fromList(availableExpr)); + break; + default: + // No arguments needed for other functions + break; + } + + // Generate partition by clause + List partitionBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + partitionBy.add(Randomly.fromList(availableExpr)); + } + } + + // Generate order by clause + List orderBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + orderBy.add(new PostgresOrderByTerm(Randomly.fromList(availableExpr), + Randomly.getBoolean())); + } + } + + // Generate window frame + WindowFrame frame = null; + if (Randomly.getBoolean()) { + WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); + PostgresExpression startExpr = PostgresExpressionGenerator.generateConstant(globalState.getRandomly()); + PostgresExpression endExpr = PostgresExpressionGenerator.generateConstant(globalState.getRandomly()); + frame = new WindowFrame(frameType, startExpr, endExpr); + } + + WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + + // Determine return type based on function + PostgresDataType returnType; + switch (functionName) { + case "percent_rank": + case "cume_dist": + returnType = PostgresDataType.FLOAT; + break; + default: + returnType = PostgresDataType.INT; + break; + } + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } +} \ No newline at end of file From 0c46e61ee9d157a775c16bcc044f64c3a26d3b77 Mon Sep 17 00:00:00 2001 From: Victor Ma Date: Wed, 12 Mar 2025 17:57:11 -0400 Subject: [PATCH 635/856] Bump duckdb version to 1.2.0 --- pom.xml | 2 +- src/sqlancer/duckdb/DuckDBErrors.java | 2 +- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 2c616138a..836d84fa5 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 1.1.3 + 1.2.0 com.facebook.presto diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index 5141ca0b1..f5cf3eb94 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -115,7 +115,7 @@ public static List getInsertErrors() { errors.addAll(getFunctionErrors()); errors.add("NOT NULL constraint failed"); - errors.add("PRIMARY KEY or UNIQUE constraint violated"); + errors.add("PRIMARY KEY or UNIQUE constraint violation"); errors.add("Duplicate key"); errors.add("can't be cast because the value is out of range for the destination type"); errors.add("Could not convert string"); diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 80bbfd61c..597fda19f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -42,7 +42,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(")"); errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { - errors.add("Cannot create an index on the rowid!"); + errors.add("cannot create an index on the rowid"); } return new SQLQueryAdapter(sb.toString(), errors, true); } From ecc9a4a388066786adafb4e4c94b879ad0f00dd0 Mon Sep 17 00:00:00 2001 From: Milan P Samuel Date: Thu, 13 Mar 2025 15:04:12 +0000 Subject: [PATCH 636/856] updated version of mariadb from 11.3.2 to 11.7.2 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91188369b..373ea3896 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -322,7 +322,7 @@ jobs: runs-on: ubuntu-latest services: mysql: - image: mariadb:11.3.2 + image: mariadb:11.7.2 env: MYSQL_ROOT_PASSWORD: root ports: From d399dfcb7a8ad4c02ba320c33ca12e7e68fdefe9 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Thu, 13 Mar 2025 15:41:02 -0400 Subject: [PATCH 637/856] Refactor Window Function Generation For Better Readability Adjust probabilities for a lower query generation in PostgreSQL window functions --- .../gen/PostgresExpressionGenerator.java | 103 +++++++++++------- .../gen/PostgresWindowFunctionGenerator.java | 83 +++++++++----- 2 files changed, 118 insertions(+), 68 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 29b02e484..9cddda042 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -424,48 +424,65 @@ private PostgresExpression generateTextExpression(int depth) { } private PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { - // Generate window function arguments - List arguments = new ArrayList<>(); - if (Randomly.getBoolean()) { - arguments.add(generateExpression(depth + 1)); - } + List arguments = generateWindowFunctionArguments(depth); + List partitionBy = generatePartitionByExpressions(depth); + List orderBy = generateOrderByExpressions(depth); + WindowFrame frame = generateWindowFrame(); + + WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + String functionName = selectWindowFunctionName(); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); +} - // Generate PARTITION BY expressions - List partitionBy = new ArrayList<>(); - if (Randomly.getBoolean()) { - int count = Randomly.smallNumber(); - for (int i = 0; i < count; i++) { - partitionBy.add(generateExpression(depth + 1)); + private List generateWindowFunctionArguments(int depth) { + List arguments = new ArrayList<>(); + if (Randomly.getBoolean()) { + arguments.add(generateExpression(depth + 1)); } + return arguments; } - // Generate ORDER BY expressions - List orderBy = new ArrayList<>(); - if (Randomly.getBoolean()) { - int count = Randomly.smallNumber(); - for (int i = 0; i < count; i++) { - PostgresExpression expr = generateExpression(depth + 1); - orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + private List generatePartitionByExpressions(int depth) { + List partitionBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + partitionBy.add(generateExpression(depth + 1)); + } } + return partitionBy; } - // Generate window frame - WindowFrame frame = null; - if (Randomly.getBoolean()) { - WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); - PostgresExpression startExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); - PostgresExpression endExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); - frame = new WindowFrame(frameType, startExpr, endExpr); + private List generateOrderByExpressions(int depth) { + List orderBy = new ArrayList<>(); + if (Randomly.getBoolean()) { + int count = Randomly.smallNumber(); + for (int i = 0; i < count; i++) { + PostgresExpression expr = generateExpression(depth + 1); + orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + } + } + return orderBy; } - WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + private WindowFrame generateWindowFrame() { + if (Randomly.getBoolean()) { + WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); + PostgresExpression startExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + PostgresExpression endExpr = generateConstant(globalState.getRandomly(), PostgresDataType.INT); + return new WindowFrame(frameType, startExpr, endExpr); + } + return null; + } - // Select a window function - String functionName = Randomly.fromList(Arrays.asList( - "row_number", "rank", "dense_rank", "percent_rank", - "cume_dist", "ntile", "lag", "lead", "first_value", - "last_value", "nth_value" - )); + private String selectWindowFunctionName() { + return Randomly.fromList(Arrays.asList( + "row_number", "rank", "dense_rank", "percent_rank", + "cume_dist", "ntile", "lag", "lead", "first_value", + "last_value", "nth_value" + )); + } return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); } @@ -716,21 +733,25 @@ public PostgresExpression generateBooleanExpression() { public PostgresSelect generateSelect() { PostgresSelect select = new PostgresSelect(); - // Add window functions to fetch columns if appropriate - if (Randomly.getBoolean()) { - List windowFunctions = new ArrayList<>(); - int numWindowFunctions = Randomly.smallNumber(); - for (int i = 0; i < numWindowFunctions; i++) { - windowFunctions.add(generateWindowFunction(0, Randomly.fromList(Arrays.asList( - PostgresDataType.INT, PostgresDataType.FLOAT - )))); - } + if (Randomly.getBooleanWithRatherLowProbability()) { + List windowFunctions = generateWindowFunctions(); select.setWindowFunctions(windowFunctions); } return select; } + private List generateWindowFunctions() { + List windowFunctions = new ArrayList<>(); + int numWindowFunctions = Randomly.smallNumber(); + for (int i = 0; i < numWindowFunctions; i++) { + windowFunctions.add(generateWindowFunction(0, Randomly.fromList(Arrays.asList( + PostgresDataType.INT, PostgresDataType.FLOAT + )))); + } + return windowFunctions; + } + @Override public List getRandomJoinClauses() { List joinStatements = new ArrayList<>(); diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java index 7537db9cb..ef9b1ca30 100644 --- a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -24,10 +24,22 @@ public class PostgresWindowFunctionGenerator { public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState globalState, List availableExpr) { - String functionName = Randomly.fromList(WINDOW_FUNCTIONS); + String functionName = selectRandomWindowFunction(); + List arguments = generateFunctionArguments(functionName, globalState, availableExpr); + WindowSpecification windowSpec = generateWindowSpecification(globalState, availableExpr); + PostgresDataType returnType = determineReturnType(functionName); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } + + private static String selectRandomWindowFunction() { + return Randomly.fromList(WINDOW_FUNCTIONS); + } + + private static List generateFunctionArguments(String functionName, + PostgresGlobalState globalState, List availableExpr) { List arguments = new ArrayList<>(); - // Generate function arguments based on function name switch (functionName) { case "ntile": arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); @@ -44,53 +56,70 @@ public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState case "last_value": arguments.add(Randomly.fromList(availableExpr)); break; - default: - // No arguments needed for other functions - break; } + + return arguments; + } - // Generate partition by clause + private static WindowSpecification generateWindowSpecification(PostgresGlobalState globalState, + List availableExpr) { + List partitionBy = generatePartitionByClause(availableExpr); + List orderBy = generateOrderByClause(availableExpr); + WindowFrame frame = generateWindowFrame(globalState); + + return new WindowSpecification(partitionBy, orderBy, frame); + } + + private static List generatePartitionByClause(List availableExpr) { List partitionBy = new ArrayList<>(); - if (Randomly.getBoolean()) { + if (Randomly.getBooleanWithRatherLowProbability()) { int count = Randomly.smallNumber(); for (int i = 0; i < count; i++) { partitionBy.add(Randomly.fromList(availableExpr)); } } + return partitionBy; + } - // Generate order by clause + private static List generateOrderByClause(List availableExpr) { List orderBy = new ArrayList<>(); - if (Randomly.getBoolean()) { + if (Randomly.getBooleanWithRatherLowProbability()) { int count = Randomly.smallNumber(); for (int i = 0; i < count; i++) { - orderBy.add(new PostgresOrderByTerm(Randomly.fromList(availableExpr), - Randomly.getBoolean())); + orderBy.add(new PostgresOrderByTerm( + Randomly.fromList(availableExpr), + Randomly.getBoolean() + )); } } + return orderBy; + } - // Generate window frame - WindowFrame frame = null; - if (Randomly.getBoolean()) { + private static WindowFrame generateWindowFrame(PostgresGlobalState globalState) { + if (Randomly.getBooleanWithRatherLowProbability()) { WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); - PostgresExpression startExpr = PostgresExpressionGenerator.generateConstant(globalState.getRandomly()); - PostgresExpression endExpr = PostgresExpressionGenerator.generateConstant(globalState.getRandomly()); - frame = new WindowFrame(frameType, startExpr, endExpr); + PostgresExpression startExpr = generateFrameBound(globalState); + PostgresExpression endExpr = generateFrameBound(globalState); + return new WindowFrame(frameType, startExpr, endExpr); } + return null; + } - WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); - - // Determine return type based on function - PostgresDataType returnType; + private static PostgresExpression generateFrameBound(PostgresGlobalState globalState) { + if (Randomly.getBooleanWithRatherLowProbability()) { + return PostgresConstant.createIntConstant(0); // CURRENT ROW + } else { + return PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT); + } + } + + private static PostgresDataType determineReturnType(String functionName) { switch (functionName) { case "percent_rank": case "cume_dist": - returnType = PostgresDataType.FLOAT; - break; + return PostgresDataType.FLOAT; default: - returnType = PostgresDataType.INT; - break; + return PostgresDataType.INT; } - - return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); } } \ No newline at end of file From d70a9a67cb4bfb8e4c5dbf42d8713308e5734dce Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 15 Mar 2025 10:49:19 -0400 Subject: [PATCH 638/856] Fix compilation errors and warnings Add more support for PostgresOrderByTerm --- .../PostgresExpectedValueVisitor.java | 22 +++++--- .../postgres/PostgresToStringVisitor.java | 52 +++++++++++++++---- src/sqlancer/postgres/PostgresVisitor.java | 1 + .../postgres/ast/PostgresOrderByTerm.java | 34 +++++++++--- src/sqlancer/postgres/ast/PostgresSelect.java | 10 +++- .../postgres/ast/PostgresWindowFunction.java | 52 +++++++++---------- .../gen/PostgresExpressionGenerator.java | 40 +++++--------- .../gen/PostgresWindowFunctionGenerator.java | 34 ++++++++---- 8 files changed, 154 insertions(+), 91 deletions(-) diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index efc1b7442..016576af3 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -22,6 +22,8 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.IgnoreMeException; +import sqlancer.postgres.ast.PostgresWindowFunction; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -42,13 +44,13 @@ private void print(PostgresExpression expr) { // @Override // public void visit(PostgresExpression expr) { - // nrTabs++; - // try { - // super.visit(expr); - // } catch (IgnoreMeException e) { - // - // } - // nrTabs--; + // nrTabs++; + // try { + // super.visit(expr); + // } catch (IgnoreMeException e) { + + // } + // nrTabs--; // } @Override @@ -96,6 +98,12 @@ public void visit(PostgresOrderByTerm op) { } + + @Override + public void visit(PostgresWindowFunction windowFunction) { + throw new IgnoreMeException(); + } + @Override public void visit(PostgresFunction f) { print(f); diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index b746bc733..e3cf46c7e 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -1,6 +1,7 @@ package sqlancer.postgres; import java.util.Optional; +import java.util.List; import sqlancer.Randomly; import sqlancer.common.visitor.BinaryOperation; @@ -30,6 +31,9 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; public final class PostgresToStringVisitor extends ToStringVisitor implements PostgresVisitor { @@ -185,13 +189,19 @@ public void visit(PostgresSelect s) { } } + // @Override + // public void visit(PostgresOrderByTerm op) { + // visit(op.getExpr()); + // sb.append(" "); + // sb.append(op.getOrder()); + // } + @Override - public void visit(PostgresOrderByTerm op) { - visit(op.getExpr()); - sb.append(" "); - sb.append(op.getOrder()); + public void visit(PostgresOrderByTerm term) { + visit(term.getExpr()); + sb.append(term.isAscending() ? " ASC" : " DESC"); } - + @Override public void visit(PostgresFunction f) { sb.append(f.getFunctionName()); @@ -366,13 +376,14 @@ public void visit(PostgresLikeOperation op) { public void visit(PostgresWindowFunction windowFunction) { sb.append(windowFunction.getFunctionName()); sb.append("("); - visit(windowFunction.getArguments()); + // Fix: Use visitList instead of visit for a list of expressions + visitList(windowFunction.getArguments()); sb.append(") OVER ("); WindowSpecification spec = windowFunction.getWindowSpec(); if (!spec.getPartitionBy().isEmpty()) { sb.append("PARTITION BY "); - visit(spec.getPartitionBy()); + visitList(spec.getPartitionBy()); } if (!spec.getOrderBy().isEmpty()) { @@ -380,9 +391,10 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(" "); } sb.append("ORDER BY "); - visit(spec.getOrderBy()); + // Fix: Create a method to handle order by terms specifically + visitOrderByList(spec.getOrderBy()); } - + if (spec.getFrame() != null) { sb.append(" "); WindowFrame frame = spec.getFrame(); @@ -395,4 +407,26 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(")"); } + + // this method handles lists of expressions + private void visitList(List expressions) { + int i = 0; + for (PostgresExpression expr : expressions) { + if (i++ != 0) { + sb.append(", "); + } + visit(expr); + } + } + + // this method handles lists of order by terms + private void visitOrderByList(List orderBy) { + int i = 0; + for (PostgresOrderByTerm term : orderBy) { + if (i++ != 0) { + sb.append(", "); + } + visit(term); + } + } } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 5951b1893..4e563824c 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -2,6 +2,7 @@ import java.util.List; +import sqlancer.postgres.ast.PostgresWindowFunction; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; diff --git a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java index 20f93536e..3c4f39fb7 100644 --- a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java +++ b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java @@ -4,9 +4,9 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresOrderByTerm implements PostgresExpression { - - private final PostgresOrder order; + private final PostgresExpression expr; + private final PostgresOrder order; public enum PostgresOrder { ASC, DESC; @@ -17,18 +17,34 @@ public static PostgresOrder getRandomOrder() { } public PostgresOrderByTerm(PostgresExpression expr, PostgresOrder order) { + if (expr == null) { + throw new IllegalArgumentException("Expression cannot be null"); + } this.expr = expr; this.order = order; } - - public PostgresOrder getOrder() { - return order; + + // Constructor for window functions, might be removed in the future to have only one constructor + public PostgresOrderByTerm(PostgresExpression expr, boolean ascending) { + if (expr == null) { + throw new IllegalArgumentException("Expression cannot be null"); + } + this.expr = expr; + this.order = ascending ? PostgresOrder.ASC : PostgresOrder.DESC; } public PostgresExpression getExpr() { return expr; } + public PostgresOrder getOrder() { + return order; + } + + public boolean isAscending() { + return order == PostgresOrder.ASC; + } + @Override public PostgresConstant getExpectedValue() { throw new AssertionError(this); @@ -38,5 +54,9 @@ public PostgresConstant getExpectedValue() { public PostgresDataType getExpressionType() { return null; } - -} + + @Override + public String toString() { + return String.format("%s %s", expr, order); + } +} \ No newline at end of file diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index 4b7773214..a51c85e72 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -2,6 +2,9 @@ import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.ArrayList; +import java.util.HashMap; import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; @@ -10,6 +13,7 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; public class PostgresSelect extends SelectBase implements PostgresExpression, Select { @@ -18,6 +22,8 @@ public class PostgresSelect extends SelectBase private List joinClauses = Collections.emptyList(); private PostgresExpression distinctOnClause; private ForClause forClause; + private List windowFunctions = new ArrayList<>(); + private Map windowDefinitions = new HashMap<>(); public enum ForClause { UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); @@ -69,7 +75,7 @@ public List getWindowFunctions() { } public void setWindowFunctions(List windowFunctions) { - this.windowFunctions = windowFunctions != null ? windowFunctions : new ArrayList<>(); + this.windowFunctions = windowFunctions; } // Add methods for window definitions @@ -191,4 +197,4 @@ public ForClause getForClause() { public String asString() { return PostgresVisitor.asString(this); } -} +} \ No newline at end of file diff --git a/src/sqlancer/postgres/ast/PostgresWindowFunction.java b/src/sqlancer/postgres/ast/PostgresWindowFunction.java index dd92abbae..71ac6853a 100644 --- a/src/sqlancer/postgres/ast/PostgresWindowFunction.java +++ b/src/sqlancer/postgres/ast/PostgresWindowFunction.java @@ -1,8 +1,6 @@ package sqlancer.postgres.ast; import java.util.List; - -import sqlancer.Randomly; import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresWindowFunction implements PostgresExpression { @@ -12,6 +10,31 @@ public class PostgresWindowFunction implements PostgresExpression { private final WindowSpecification windowSpec; private final PostgresDataType returnType; + public PostgresWindowFunction(String functionName, List arguments, + WindowSpecification windowSpec, PostgresDataType returnType) { + this.functionName = functionName; + this.arguments = arguments; + this.windowSpec = windowSpec; + this.returnType = returnType; + } + + public String getFunctionName() { + return functionName; + } + + public List getArguments() { + return arguments; + } + + public WindowSpecification getWindowSpec() { + return windowSpec; + } + + @Override + public PostgresDataType getExpressionType() { + return returnType; + } + public static class WindowSpecification { private final List partitionBy; private final List orderBy; @@ -76,29 +99,4 @@ public PostgresExpression getEndExpr() { return endExpr; } } - - public PostgresWindowFunction(String functionName, List arguments, - WindowSpecification windowSpec, PostgresDataType returnType) { - this.functionName = functionName; - this.arguments = arguments; - this.windowSpec = windowSpec; - this.returnType = returnType; - } - - public String getFunctionName() { - return functionName; - } - - public List getArguments() { - return arguments; - } - - public WindowSpecification getWindowSpec() { - return windowSpec; - } - - @Override - public PostgresDataType getExpressionType() { - return returnType; - } } \ No newline at end of file diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 9cddda042..83f9096d0 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -52,7 +52,7 @@ import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; -import sqlancer.postgres.ast.PostgresOrderByTerm.PostgresOrder; +// import sqlancer.postgres.ast.PostgresOrderByTerm.PostgresOrder; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression.POSIXRegex; import sqlancer.postgres.ast.PostgresPostfixOperation; @@ -67,6 +67,10 @@ import sqlancer.postgres.ast.PostgresSelect.SelectType; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; +import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; + public class PostgresExpressionGenerator implements ExpressionGenerator, NoRECGenerator, @@ -92,7 +96,7 @@ public class PostgresExpressionGenerator implements ExpressionGenerator functionsAndTypes; private final List allowedFunctionTypes; - + public PostgresExpressionGenerator(PostgresGlobalState globalState) { this.r = globalState.getRandomly(); this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); @@ -119,8 +123,8 @@ public PostgresExpression generateExpression(int depth) { public List generateOrderBys() { List orderBys = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber(); i++) { - orderBys.add(new PostgresOrderByTerm(PostgresColumnValue.create(Randomly.fromList(columns), null), - PostgresOrder.getRandomOrder())); + PostgresExpression expr = PostgresColumnValue.create(Randomly.fromList(columns), null); + orderBys.add(expr); } return orderBys; } @@ -442,7 +446,6 @@ private List generateWindowFunctionArguments(int depth) { } return arguments; } - private List generatePartitionByExpressions(int depth) { List partitionBy = new ArrayList<>(); if (Randomly.getBoolean()) { @@ -460,6 +463,7 @@ private List generateOrderByExpressions(int depth) { int count = Randomly.smallNumber(); for (int i = 0; i < count; i++) { PostgresExpression expr = generateExpression(depth + 1); + // Call the second constructor in PostgresOrderByTerm, might be removed in the future to have only one constructor orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); } } @@ -484,8 +488,6 @@ private String selectWindowFunctionName() { )); } - return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); - } private PostgresExpression generateConcat(int depth) { PostgresExpression left = generateExpression(depth + 1, PostgresDataType.TEXT); @@ -511,7 +513,7 @@ private PostgresExpression generateBitExpression(int depth) { } private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION } private PostgresExpression generateIntExpression(int depth) { @@ -529,11 +531,12 @@ private PostgresExpression generateIntExpression(int depth) { case BINARY_ARITHMETIC_EXPRESSION: return new PostgresBinaryArithmeticOperation(generateExpression(depth + 1, PostgresDataType.INT), generateExpression(depth + 1, PostgresDataType.INT), PostgresBinaryOperator.getRandom()); + case WINDOW_FUNCTION: + return generateWindowFunction(depth + 1, PostgresDataType.INT); default: throw new AssertionError(); } } - private PostgresExpression createColumnOfType(PostgresDataType type) { List columns = filterColumns(type); PostgresColumn fromList = Randomly.fromList(columns); @@ -831,25 +834,6 @@ public String generateUnoptimizedQueryString(PostgresSelect select, PostgresExpr public String generateExplainQuery(PostgresSelect select) { return "EXPLAIN " + select.asString(); } - - @Override - public boolean mutate(PostgresSelect select) { - List> mutators = new ArrayList<>(); - - mutators.add(this::mutateJoin); - mutators.add(this::mutateWhere); - mutators.add(this::mutateGroupBy); - mutators.add(this::mutateHaving); - if (!PostgresBugs.bug18643) { - mutators.add(this::mutateAnd); - mutators.add(this::mutateOr); - } - // mutators.add(this::mutateLimit); - mutators.add(this::mutateDistinct); - - return Randomly.fromList(mutators).apply(select); - } - @Override public boolean mutate(PostgresSelect select) { diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java index ef9b1ca30..1b9462d2b 100644 --- a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -6,6 +6,7 @@ import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresOrderByTerm; @@ -14,7 +15,8 @@ import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; public class PostgresWindowFunctionGenerator { - + + private static final List WINDOW_FUNCTIONS = Arrays.asList( "row_number", "rank", "dense_rank", "percent_rank", "cume_dist", "ntile", "lag", "lead", "first_value", @@ -23,7 +25,7 @@ public class PostgresWindowFunctionGenerator { public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState globalState, List availableExpr) { - + String functionName = selectRandomWindowFunction(); List arguments = generateFunctionArguments(functionName, globalState, availableExpr); WindowSpecification windowSpec = generateWindowSpecification(globalState, availableExpr); @@ -39,17 +41,18 @@ private static String selectRandomWindowFunction() { private static List generateFunctionArguments(String functionName, PostgresGlobalState globalState, List availableExpr) { List arguments = new ArrayList<>(); - + switch (functionName) { case "ntile": - arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); + + arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); break; case "lag": case "lead": case "nth_value": arguments.add(Randomly.fromList(availableExpr)); if (Randomly.getBoolean()) { - arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly())); + arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); } break; case "first_value": @@ -57,7 +60,7 @@ private static List generateFunctionArguments(String functio arguments.add(Randomly.fromList(availableExpr)); break; } - + return arguments; } @@ -66,7 +69,7 @@ private static WindowSpecification generateWindowSpecification(PostgresGlobalSta List partitionBy = generatePartitionByClause(availableExpr); List orderBy = generateOrderByClause(availableExpr); WindowFrame frame = generateWindowFrame(globalState); - + return new WindowSpecification(partitionBy, orderBy, frame); } @@ -94,7 +97,6 @@ private static List generateOrderByClause(List Date: Sun, 16 Mar 2025 01:07:28 +0530 Subject: [PATCH 639/856] Added IllegalArgumentException to handle the possible exception. --- src/sqlancer/common/query/ExpectedErrors.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index f71d185e1..38d5c9757 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -43,11 +43,17 @@ public ExpectedErrors addRegex(Pattern errorPattern) { } public ExpectedErrors addRegexString(String errorPattern) { + if (errorPattern == null) { + throw new IllegalArgumentException(); + } regexes.add(Pattern.compile(errorPattern)); return this; } public ExpectedErrors addAll(Collection list) { + if (list == null) { + throw new IllegalArgumentException(); + } errors.addAll(list); return this; } From 9bf6f1b7b0fb32d4237f00ed560843274eb9f5bf Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 15 Mar 2025 17:33:35 -0400 Subject: [PATCH 640/856] Fix style checks --- .../PostgresExpectedValueVisitor.java | 17 +++-- .../postgres/PostgresToStringVisitor.java | 18 ++--- src/sqlancer/postgres/PostgresVisitor.java | 5 +- .../postgres/ast/PostgresOrderByTerm.java | 10 +-- src/sqlancer/postgres/ast/PostgresSelect.java | 14 ++-- .../postgres/ast/PostgresWindowFunction.java | 17 +++-- .../gen/PostgresExpressionGenerator.java | 51 +++++++------ .../gen/PostgresWindowFunctionGenerator.java | 71 ++++++++++--------- 8 files changed, 98 insertions(+), 105 deletions(-) diff --git a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java index 016576af3..8985caf0b 100644 --- a/src/sqlancer/postgres/PostgresExpectedValueVisitor.java +++ b/src/sqlancer/postgres/PostgresExpectedValueVisitor.java @@ -1,5 +1,6 @@ package sqlancer.postgres; +import sqlancer.IgnoreMeException; import sqlancer.postgres.ast.PostgresAggregate; import sqlancer.postgres.ast.PostgresBetweenOperation; import sqlancer.postgres.ast.PostgresBinaryLogicalOperation; @@ -22,7 +23,6 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; -import sqlancer.IgnoreMeException; import sqlancer.postgres.ast.PostgresWindowFunction; public final class PostgresExpectedValueVisitor implements PostgresVisitor { @@ -44,13 +44,13 @@ private void print(PostgresExpression expr) { // @Override // public void visit(PostgresExpression expr) { - // nrTabs++; - // try { - // super.visit(expr); - // } catch (IgnoreMeException e) { - - // } - // nrTabs--; + // nrTabs++; + // try { + // super.visit(expr); + // } catch (IgnoreMeException e) { + + // } + // nrTabs--; // } @Override @@ -98,7 +98,6 @@ public void visit(PostgresOrderByTerm op) { } - @Override public void visit(PostgresWindowFunction windowFunction) { throw new IgnoreMeException(); diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index e3cf46c7e..5df87aa5a 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -1,7 +1,7 @@ package sqlancer.postgres; -import java.util.Optional; import java.util.List; +import java.util.Optional; import sqlancer.Randomly; import sqlancer.common.visitor.BinaryOperation; @@ -191,9 +191,9 @@ public void visit(PostgresSelect s) { // @Override // public void visit(PostgresOrderByTerm op) { - // visit(op.getExpr()); - // sb.append(" "); - // sb.append(op.getOrder()); + // visit(op.getExpr()); + // sb.append(" "); + // sb.append(op.getOrder()); // } @Override @@ -201,7 +201,7 @@ public void visit(PostgresOrderByTerm term) { visit(term.getExpr()); sb.append(term.isAscending() ? " ASC" : " DESC"); } - + @Override public void visit(PostgresFunction f) { sb.append(f.getFunctionName()); @@ -379,13 +379,13 @@ public void visit(PostgresWindowFunction windowFunction) { // Fix: Use visitList instead of visit for a list of expressions visitList(windowFunction.getArguments()); sb.append(") OVER ("); - + WindowSpecification spec = windowFunction.getWindowSpec(); if (!spec.getPartitionBy().isEmpty()) { sb.append("PARTITION BY "); visitList(spec.getPartitionBy()); } - + if (!spec.getOrderBy().isEmpty()) { if (!spec.getPartitionBy().isEmpty()) { sb.append(" "); @@ -394,7 +394,7 @@ public void visit(PostgresWindowFunction windowFunction) { // Fix: Create a method to handle order by terms specifically visitOrderByList(spec.getOrderBy()); } - + if (spec.getFrame() != null) { sb.append(" "); WindowFrame frame = spec.getFrame(); @@ -404,7 +404,7 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(" AND "); visit(frame.getEndExpr()); } - + sb.append(")"); } diff --git a/src/sqlancer/postgres/PostgresVisitor.java b/src/sqlancer/postgres/PostgresVisitor.java index 4e563824c..d5bf71e81 100644 --- a/src/sqlancer/postgres/PostgresVisitor.java +++ b/src/sqlancer/postgres/PostgresVisitor.java @@ -2,7 +2,6 @@ import java.util.List; -import sqlancer.postgres.ast.PostgresWindowFunction; import sqlancer.postgres.PostgresSchema.PostgresColumn; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.ast.PostgresAggregate; @@ -27,6 +26,7 @@ import sqlancer.postgres.ast.PostgresSelect.PostgresSubquery; import sqlancer.postgres.ast.PostgresSimilarTo; import sqlancer.postgres.ast.PostgresTableReference; +import sqlancer.postgres.ast.PostgresWindowFunction; import sqlancer.postgres.gen.PostgresExpressionGenerator; public interface PostgresVisitor { @@ -118,8 +118,7 @@ default void visit(PostgresExpression expression) { visit((PostgresTableReference) expression); } else if (expression instanceof PostgresWindowFunction) { visit((PostgresWindowFunction) expression); - } - else { + } else { throw new AssertionError(expression); } } diff --git a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java index 3c4f39fb7..96debf741 100644 --- a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java +++ b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java @@ -4,7 +4,7 @@ import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresOrderByTerm implements PostgresExpression { - + private final PostgresExpression expr; private final PostgresOrder order; @@ -23,7 +23,7 @@ public PostgresOrderByTerm(PostgresExpression expr, PostgresOrder order) { this.expr = expr; this.order = order; } - + // Constructor for window functions, might be removed in the future to have only one constructor public PostgresOrderByTerm(PostgresExpression expr, boolean ascending) { if (expr == null) { @@ -40,7 +40,7 @@ public PostgresExpression getExpr() { public PostgresOrder getOrder() { return order; } - + public boolean isAscending() { return order == PostgresOrder.ASC; } @@ -54,9 +54,9 @@ public PostgresConstant getExpectedValue() { public PostgresDataType getExpressionType() { return null; } - + @Override public String toString() { return String.format("%s %s", expr, order); } -} \ No newline at end of file +} diff --git a/src/sqlancer/postgres/ast/PostgresSelect.java b/src/sqlancer/postgres/ast/PostgresSelect.java index a51c85e72..1d7172e3e 100644 --- a/src/sqlancer/postgres/ast/PostgresSelect.java +++ b/src/sqlancer/postgres/ast/PostgresSelect.java @@ -1,10 +1,10 @@ package sqlancer.postgres.ast; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.ArrayList; -import java.util.HashMap; import sqlancer.Randomly; import sqlancer.common.ast.SelectBase; @@ -23,7 +23,7 @@ public class PostgresSelect extends SelectBase private PostgresExpression distinctOnClause; private ForClause forClause; private List windowFunctions = new ArrayList<>(); - private Map windowDefinitions = new HashMap<>(); + private final Map windowDefinitions = new HashMap<>(); public enum ForClause { UPDATE("UPDATE"), NO_KEY_UPDATE("NO KEY UPDATE"), SHARE("SHARE"), KEY_SHARE("KEY SHARE"); @@ -48,9 +48,8 @@ public static class WindowDefinition { private final List orderBy; private final WindowFrame frame; - public WindowDefinition(List partitionBy, - List orderBy, - WindowFrame frame) { + public WindowDefinition(List partitionBy, List orderBy, + WindowFrame frame) { this.partitionBy = partitionBy; this.orderBy = orderBy; this.frame = frame; @@ -91,7 +90,6 @@ public Map getWindowDefinitions() { return windowDefinitions; } - public static class PostgresFromTable implements PostgresExpression { private final PostgresTable t; private final boolean only; @@ -197,4 +195,4 @@ public ForClause getForClause() { public String asString() { return PostgresVisitor.asString(this); } -} \ No newline at end of file +} diff --git a/src/sqlancer/postgres/ast/PostgresWindowFunction.java b/src/sqlancer/postgres/ast/PostgresWindowFunction.java index 71ac6853a..15f87364f 100644 --- a/src/sqlancer/postgres/ast/PostgresWindowFunction.java +++ b/src/sqlancer/postgres/ast/PostgresWindowFunction.java @@ -1,17 +1,18 @@ package sqlancer.postgres.ast; import java.util.List; + import sqlancer.postgres.PostgresSchema.PostgresDataType; public class PostgresWindowFunction implements PostgresExpression { - + private final String functionName; private final List arguments; private final WindowSpecification windowSpec; private final PostgresDataType returnType; - public PostgresWindowFunction(String functionName, List arguments, - WindowSpecification windowSpec, PostgresDataType returnType) { + public PostgresWindowFunction(String functionName, List arguments, + WindowSpecification windowSpec, PostgresDataType returnType) { this.functionName = functionName; this.arguments = arguments; this.windowSpec = windowSpec; @@ -40,9 +41,8 @@ public static class WindowSpecification { private final List orderBy; private final WindowFrame frame; - public WindowSpecification(List partitionBy, - List orderBy, - WindowFrame frame) { + public WindowSpecification(List partitionBy, List orderBy, + WindowFrame frame) { this.partitionBy = partitionBy; this.orderBy = orderBy; this.frame = frame; @@ -63,8 +63,7 @@ public WindowFrame getFrame() { public static class WindowFrame { public enum FrameType { - ROWS("ROWS"), - RANGE("RANGE"); + ROWS("ROWS"), RANGE("RANGE"); private final String sql; @@ -99,4 +98,4 @@ public PostgresExpression getEndExpr() { return endExpr; } } -} \ No newline at end of file +} diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 83f9096d0..9f3251b1d 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -71,7 +71,6 @@ import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; - public class PostgresExpressionGenerator implements ExpressionGenerator, NoRECGenerator, TLPWhereGenerator, @@ -96,7 +95,7 @@ public class PostgresExpressionGenerator implements ExpressionGenerator functionsAndTypes; private final List allowedFunctionTypes; - + public PostgresExpressionGenerator(PostgresGlobalState globalState) { this.r = globalState.getRandomly(); this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); @@ -426,18 +425,18 @@ private PostgresExpression generateTextExpression(int depth) { throw new AssertionError(); } } - + private PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { - List arguments = generateWindowFunctionArguments(depth); - List partitionBy = generatePartitionByExpressions(depth); - List orderBy = generateOrderByExpressions(depth); - WindowFrame frame = generateWindowFrame(); - - WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); - String functionName = selectWindowFunctionName(); - - return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); -} + List arguments = generateWindowFunctionArguments(depth); + List partitionBy = generatePartitionByExpressions(depth); + List orderBy = generateOrderByExpressions(depth); + WindowFrame frame = generateWindowFrame(); + + WindowSpecification windowSpec = new WindowSpecification(partitionBy, orderBy, frame); + String functionName = selectWindowFunctionName(); + + return new PostgresWindowFunction(functionName, arguments, windowSpec, returnType); + } private List generateWindowFunctionArguments(int depth) { List arguments = new ArrayList<>(); @@ -446,6 +445,7 @@ private List generateWindowFunctionArguments(int depth) { } return arguments; } + private List generatePartitionByExpressions(int depth) { List partitionBy = new ArrayList<>(); if (Randomly.getBoolean()) { @@ -463,7 +463,8 @@ private List generateOrderByExpressions(int depth) { int count = Randomly.smallNumber(); for (int i = 0; i < count; i++) { PostgresExpression expr = generateExpression(depth + 1); - // Call the second constructor in PostgresOrderByTerm, might be removed in the future to have only one constructor + // Call the second constructor in PostgresOrderByTerm, might be removed in the future to have only one + // constructor orderBy.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); } } @@ -481,14 +482,10 @@ private WindowFrame generateWindowFrame() { } private String selectWindowFunctionName() { - return Randomly.fromList(Arrays.asList( - "row_number", "rank", "dense_rank", "percent_rank", - "cume_dist", "ntile", "lag", "lead", "first_value", - "last_value", "nth_value" - )); + return Randomly.fromList(Arrays.asList("row_number", "rank", "dense_rank", "percent_rank", "cume_dist", "ntile", + "lag", "lead", "first_value", "last_value", "nth_value")); } - private PostgresExpression generateConcat(int depth) { PostgresExpression left = generateExpression(depth + 1, PostgresDataType.TEXT); PostgresExpression right = generateExpression(depth + 1); @@ -513,7 +510,7 @@ private PostgresExpression generateBitExpression(int depth) { } private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION } private PostgresExpression generateIntExpression(int depth) { @@ -537,6 +534,7 @@ private PostgresExpression generateIntExpression(int depth) { throw new AssertionError(); } } + private PostgresExpression createColumnOfType(PostgresDataType type) { List columns = filterColumns(type); PostgresColumn fromList = Randomly.fromList(columns); @@ -735,12 +733,12 @@ public PostgresExpression generateBooleanExpression() { @Override public PostgresSelect generateSelect() { PostgresSelect select = new PostgresSelect(); - + if (Randomly.getBooleanWithRatherLowProbability()) { List windowFunctions = generateWindowFunctions(); select.setWindowFunctions(windowFunctions); } - + return select; } @@ -748,9 +746,8 @@ private List generateWindowFunctions() { List windowFunctions = new ArrayList<>(); int numWindowFunctions = Randomly.smallNumber(); for (int i = 0; i < numWindowFunctions; i++) { - windowFunctions.add(generateWindowFunction(0, Randomly.fromList(Arrays.asList( - PostgresDataType.INT, PostgresDataType.FLOAT - )))); + windowFunctions.add(generateWindowFunction(0, + Randomly.fromList(Arrays.asList(PostgresDataType.INT, PostgresDataType.FLOAT)))); } return windowFunctions; } @@ -834,7 +831,7 @@ public String generateUnoptimizedQueryString(PostgresSelect select, PostgresExpr public String generateExplainQuery(PostgresSelect select) { return "EXPLAIN " + select.asString(); } - + @Override public boolean mutate(PostgresSelect select) { List> mutators = new ArrayList<>(); diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java index 1b9462d2b..a2aec3f7d 100644 --- a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -6,22 +6,23 @@ import sqlancer.Randomly; import sqlancer.postgres.PostgresGlobalState; -import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.ast.PostgresConstant; import sqlancer.postgres.ast.PostgresExpression; import sqlancer.postgres.ast.PostgresOrderByTerm; import sqlancer.postgres.ast.PostgresWindowFunction; import sqlancer.postgres.ast.PostgresWindowFunction.WindowFrame; import sqlancer.postgres.ast.PostgresWindowFunction.WindowSpecification; -public class PostgresWindowFunctionGenerator { +public final class PostgresWindowFunctionGenerator { + private static final List WINDOW_FUNCTIONS = Arrays.asList("row_number", "rank", "dense_rank", + "percent_rank", "cume_dist", "ntile", "lag", "lead", "first_value", "last_value", "nth_value"); - private static final List WINDOW_FUNCTIONS = Arrays.asList( - "row_number", "rank", "dense_rank", "percent_rank", - "cume_dist", "ntile", "lag", "lead", "first_value", - "last_value", "nth_value" - ); + // Private constructor to prevent instantiation + private PostgresWindowFunctionGenerator() { + throw new AssertionError("Utility class should not be instantiated"); + } public static PostgresWindowFunction generateWindowFunction(PostgresGlobalState globalState, List availableExpr) { @@ -38,27 +39,31 @@ private static String selectRandomWindowFunction() { return Randomly.fromList(WINDOW_FUNCTIONS); } - private static List generateFunctionArguments(String functionName, + private static List generateFunctionArguments(String functionName, PostgresGlobalState globalState, List availableExpr) { List arguments = new ArrayList<>(); switch (functionName) { - case "ntile": - - arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); - break; - case "lag": - case "lead": - case "nth_value": - arguments.add(Randomly.fromList(availableExpr)); - if (Randomly.getBoolean()) { - arguments.add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); - } - break; - case "first_value": - case "last_value": - arguments.add(Randomly.fromList(availableExpr)); - break; + case "ntile": + arguments + .add(PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); + break; + case "lag": + case "lead": + case "nth_value": + arguments.add(Randomly.fromList(availableExpr)); + if (Randomly.getBoolean()) { + arguments.add( + PostgresExpressionGenerator.generateConstant(globalState.getRandomly(), PostgresDataType.INT)); + } + break; + case "first_value": + case "last_value": + arguments.add(Randomly.fromList(availableExpr)); + break; + default: + // No arguments needed for other window functions + break; } return arguments; @@ -89,14 +94,12 @@ private static List generateOrderByClause(List Date: Sun, 16 Mar 2025 10:38:24 -0400 Subject: [PATCH 641/856] Remove commented code and refactor PostgresWindowFunctionGenerator and PostgresToStringVisitor --- .../postgres/PostgresToStringVisitor.java | 38 ++----------------- .../gen/PostgresExpressionGenerator.java | 1 - .../gen/PostgresWindowFunctionGenerator.java | 23 ++++------- 3 files changed, 12 insertions(+), 50 deletions(-) diff --git a/src/sqlancer/postgres/PostgresToStringVisitor.java b/src/sqlancer/postgres/PostgresToStringVisitor.java index 5df87aa5a..87bd3c429 100644 --- a/src/sqlancer/postgres/PostgresToStringVisitor.java +++ b/src/sqlancer/postgres/PostgresToStringVisitor.java @@ -189,13 +189,6 @@ public void visit(PostgresSelect s) { } } - // @Override - // public void visit(PostgresOrderByTerm op) { - // visit(op.getExpr()); - // sb.append(" "); - // sb.append(op.getOrder()); - // } - @Override public void visit(PostgresOrderByTerm term) { visit(term.getExpr()); @@ -373,17 +366,17 @@ public void visit(PostgresLikeOperation op) { } @Override + @SuppressWarnings("unchecked") public void visit(PostgresWindowFunction windowFunction) { sb.append(windowFunction.getFunctionName()); sb.append("("); - // Fix: Use visitList instead of visit for a list of expressions - visitList(windowFunction.getArguments()); + visit(windowFunction.getArguments()); sb.append(") OVER ("); WindowSpecification spec = windowFunction.getWindowSpec(); if (!spec.getPartitionBy().isEmpty()) { sb.append("PARTITION BY "); - visitList(spec.getPartitionBy()); + visit(spec.getPartitionBy()); } if (!spec.getOrderBy().isEmpty()) { @@ -391,8 +384,7 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(" "); } sb.append("ORDER BY "); - // Fix: Create a method to handle order by terms specifically - visitOrderByList(spec.getOrderBy()); + visit((List) (List) spec.getOrderBy()); } if (spec.getFrame() != null) { @@ -407,26 +399,4 @@ public void visit(PostgresWindowFunction windowFunction) { sb.append(")"); } - - // this method handles lists of expressions - private void visitList(List expressions) { - int i = 0; - for (PostgresExpression expr : expressions) { - if (i++ != 0) { - sb.append(", "); - } - visit(expr); - } - } - - // this method handles lists of order by terms - private void visitOrderByList(List orderBy) { - int i = 0; - for (PostgresOrderByTerm term : orderBy) { - if (i++ != 0) { - sb.append(", "); - } - visit(term); - } - } } diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 9f3251b1d..39a31182a 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -52,7 +52,6 @@ import sqlancer.postgres.ast.PostgresJoin.PostgresJoinType; import sqlancer.postgres.ast.PostgresLikeOperation; import sqlancer.postgres.ast.PostgresOrderByTerm; -// import sqlancer.postgres.ast.PostgresOrderByTerm.PostgresOrder; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression; import sqlancer.postgres.ast.PostgresPOSIXRegularExpression.POSIXRegex; import sqlancer.postgres.ast.PostgresPostfixOperation; diff --git a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java index a2aec3f7d..6daf66c3c 100644 --- a/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresWindowFunctionGenerator.java @@ -19,7 +19,6 @@ public final class PostgresWindowFunctionGenerator { private static final List WINDOW_FUNCTIONS = Arrays.asList("row_number", "rank", "dense_rank", "percent_rank", "cume_dist", "ntile", "lag", "lead", "first_value", "last_value", "nth_value"); - // Private constructor to prevent instantiation private PostgresWindowFunctionGenerator() { throw new AssertionError("Utility class should not be instantiated"); } @@ -72,10 +71,15 @@ private static List generateFunctionArguments(String functio private static WindowSpecification generateWindowSpecification(PostgresGlobalState globalState, List availableExpr) { List partitionBy = generatePartitionByClause(availableExpr); - List orderBy = generateOrderByClause(availableExpr); - WindowFrame frame = generateWindowFrame(globalState); + PostgresExpressionGenerator exprGen = new PostgresExpressionGenerator(globalState); + List orderBys = exprGen.generateOrderBys(); + List orderByTerms = new ArrayList<>(); + for (PostgresExpression expr : orderBys) { + orderByTerms.add(new PostgresOrderByTerm(expr, Randomly.getBoolean())); + } - return new WindowSpecification(partitionBy, orderBy, frame); + WindowFrame frame = generateWindowFrame(globalState); + return new WindowSpecification(partitionBy, orderByTerms, frame); } private static List generatePartitionByClause(List availableExpr) { @@ -89,17 +93,6 @@ private static List generatePartitionByClause(List generateOrderByClause(List availableExpr) { - List orderBy = new ArrayList<>(); - if (Randomly.getBooleanWithRatherLowProbability()) { - int count = Randomly.smallNumber(); - for (int i = 0; i < count; i++) { - orderBy.add(new PostgresOrderByTerm(Randomly.fromList(availableExpr), Randomly.getBoolean())); - } - } - return orderBy; - } - private static WindowFrame generateWindowFrame(PostgresGlobalState globalState) { if (Randomly.getBooleanWithRatherLowProbability()) { WindowFrame.FrameType frameType = Randomly.fromOptions(WindowFrame.FrameType.values()); From 89ebbf58298895117582ecf6b6bf691056917810 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 16 Mar 2025 14:58:55 -0400 Subject: [PATCH 642/856] Remove window function generation from generateIntExpression --- .../gen/PostgresExpressionGenerator.java | 20 ++++++++----------- 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java index 39a31182a..bad87affc 100644 --- a/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExpressionGenerator.java @@ -15,7 +15,6 @@ import sqlancer.common.gen.ExpressionGenerator; import sqlancer.common.gen.NoRECGenerator; import sqlancer.common.gen.TLPWhereGenerator; -import sqlancer.common.schema.AbstractTables; import sqlancer.postgres.PostgresBugs; import sqlancer.postgres.PostgresCompoundDataType; import sqlancer.postgres.PostgresGlobalState; @@ -231,8 +230,7 @@ private PostgresExpression generateBooleanExpression(int depth) { } private PostgresDataType getMeaningfulType() { - // make it more likely that the expression does not only consist of constant - // expressions + // make it more likely that the expression does not only consist of constant expressions if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { return PostgresDataType.getRandomType(); } else { @@ -425,7 +423,7 @@ private PostgresExpression generateTextExpression(int depth) { } } - private PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { + public PostgresExpression generateWindowFunction(int depth, PostgresDataType returnType) { List arguments = generateWindowFunctionArguments(depth); List partitionBy = generatePartitionByExpressions(depth); List orderBy = generateOrderByExpressions(depth); @@ -508,8 +506,9 @@ private PostgresExpression generateBitExpression(int depth) { } } + // Removed WINDOW_FUNCTION option from the integer expression generation. private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, WINDOW_FUNCTION + UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION } private PostgresExpression generateIntExpression(int depth) { @@ -527,8 +526,6 @@ private PostgresExpression generateIntExpression(int depth) { case BINARY_ARITHMETIC_EXPRESSION: return new PostgresBinaryArithmeticOperation(generateExpression(depth + 1, PostgresDataType.INT), generateExpression(depth + 1, PostgresDataType.INT), PostgresBinaryOperator.getRandom()); - case WINDOW_FUNCTION: - return generateWindowFunction(depth + 1, PostgresDataType.INT); default: throw new AssertionError(); } @@ -718,7 +715,8 @@ public PostgresExpression isNull(PostgresExpression expr) { } @Override - public PostgresExpressionGenerator setTablesAndColumns(AbstractTables targetTables) { + public PostgresExpressionGenerator setTablesAndColumns( + sqlancer.common.schema.AbstractTables targetTables) { this.targetTables = targetTables.getTables(); this.columns = targetTables.getColumns(); return this; @@ -806,7 +804,7 @@ public String generateOptimizedQueryString(PostgresSelect select, PostgresExpres select.setFetchColumns(Arrays.asList(allColumns)); } select.setWhereClause(whereCondition); - if (Randomly.getBooleanWithSmallProbability()) { + if (Randomly.getBooleanWithRatherLowProbability()) { select.setOrderByClauses(generateOrderBys()); } select.setSelectType(SelectType.ALL); @@ -839,7 +837,7 @@ public boolean mutate(PostgresSelect select) { mutators.add(this::mutateWhere); mutators.add(this::mutateGroupBy); mutators.add(this::mutateHaving); - mutators.add(this::mutateWindowFunction); // Add window function mutation + mutators.add(this::mutateWindowFunction); if (!PostgresBugs.bug18643) { mutators.add(this::mutateAnd); mutators.add(this::mutateOr); @@ -852,13 +850,11 @@ public boolean mutate(PostgresSelect select) { private boolean mutateWindowFunction(PostgresSelect select) { List windowFunctions = select.getWindowFunctions(); if (windowFunctions == null || windowFunctions.isEmpty()) { - // Add a new window function windowFunctions = new ArrayList<>(); windowFunctions.add(generateWindowFunction(0, PostgresDataType.INT)); select.setWindowFunctions(windowFunctions); return false; } else { - // Remove a random window function windowFunctions.remove(Randomly.fromList(windowFunctions)); if (windowFunctions.isEmpty()) { select.setWindowFunctions(null); From c5bc6dd654b6dfc0ea2a86238671b7f7a7cb2778 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Wed, 19 Mar 2025 14:09:23 -0400 Subject: [PATCH 643/856] Add test coverage for Randomly class --- test/sqlancer/TestRandomly.java | 56 +++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/test/sqlancer/TestRandomly.java b/test/sqlancer/TestRandomly.java index 6be6b1bd4..5a58ca50b 100644 --- a/test/sqlancer/TestRandomly.java +++ b/test/sqlancer/TestRandomly.java @@ -234,4 +234,60 @@ private List getRandomValueList(Randomly r) { return values; } + @Test + public void testGetPercentage() { + for (int i = 0; i < NR_MIN_RUNS; i++) { + double percentage = Randomly.getPercentage(); + assertTrue(percentage >= 0.0); + assertTrue(percentage <= 1.0); + } + } + + @Test + public void testGetChar() { + Randomly r = new Randomly(); + boolean encounteredAlphabetic = false; + boolean encounteredNumeric = false; + boolean encounteredSpecial = false; + int i = 0; + do { + String c = r.getChar(); + assertEquals(1, c.length()); + if (Character.isAlphabetic(c.charAt(0))) { + encounteredAlphabetic = true; + } else if (Character.isDigit(c.charAt(0))) { + encounteredNumeric = true; + } else { + encounteredSpecial = true; + } + } while (!encounteredAlphabetic || !encounteredNumeric || !encounteredSpecial || i++ < NR_MIN_RUNS); + } + + @Test + public void testGetAlphabeticChar() { + Randomly r = new Randomly(); + for (int i = 0; i < NR_MIN_RUNS; i++) { + String c = r.getAlphabeticChar(); + assertEquals(1, c.length()); + assertTrue(Character.isAlphabetic(c.charAt(0))); + } + } + + @Test + public void testGetBooleanWithSmallProbability() { + int trueCount = 0; + int totalRuns = NR_MIN_RUNS; + + for (int i = 0; i < totalRuns; i++) { + if (Randomly.getBooleanWithSmallProbability()) { + trueCount++; + } + } + + double trueRatio = (double) trueCount / totalRuns; + assertTrue(trueRatio > 0.005); + assertTrue(trueRatio < 0.015); + + } + } From e2f5495c85a0f3d919a6b9b63fa66aefb7dff26d Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Thu, 20 Mar 2025 10:01:57 -0400 Subject: [PATCH 644/856] Add assert in testGetChar --- test/sqlancer/TestRandomly.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/sqlancer/TestRandomly.java b/test/sqlancer/TestRandomly.java index 5a58ca50b..ad0899420 100644 --- a/test/sqlancer/TestRandomly.java +++ b/test/sqlancer/TestRandomly.java @@ -261,6 +261,9 @@ public void testGetChar() { encounteredSpecial = true; } } while (!encounteredAlphabetic || !encounteredNumeric || !encounteredSpecial || i++ < NR_MIN_RUNS); + assertTrue(encounteredAlphabetic, "Never encounter an alphabetic character."); + assertTrue(encounteredNumeric, "Never encounter a numeric character."); + assertTrue(encounteredSpecial, "Did not encounter a special character."); } @Test From 25048b1a10a5c2be747b25a1d3c8649969e8fd90 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Thu, 20 Mar 2025 10:04:05 -0400 Subject: [PATCH 645/856] Add assert in testGetChar --- test/sqlancer/TestRandomly.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sqlancer/TestRandomly.java b/test/sqlancer/TestRandomly.java index ad0899420..284f0083a 100644 --- a/test/sqlancer/TestRandomly.java +++ b/test/sqlancer/TestRandomly.java @@ -263,7 +263,7 @@ public void testGetChar() { } while (!encounteredAlphabetic || !encounteredNumeric || !encounteredSpecial || i++ < NR_MIN_RUNS); assertTrue(encounteredAlphabetic, "Never encounter an alphabetic character."); assertTrue(encounteredNumeric, "Never encounter a numeric character."); - assertTrue(encounteredSpecial, "Did not encounter a special character."); + assertTrue(encounteredSpecial, "Never encounter a special character."); } @Test From b8a1162e5469f4e35eefd20ca4beac42bc1a2e9f Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Thu, 20 Mar 2025 11:19:42 -0400 Subject: [PATCH 646/856] Add assertions to test methods in testRandomly --- test/sqlancer/TestRandomly.java | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/test/sqlancer/TestRandomly.java b/test/sqlancer/TestRandomly.java index 6be6b1bd4..595f0337b 100644 --- a/test/sqlancer/TestRandomly.java +++ b/test/sqlancer/TestRandomly.java @@ -38,6 +38,7 @@ public void testSubset() { boolean encounteredStrictSubsetNonEmpty = false; Integer[] options = { 1, 2, 3 }; List optionList = new ArrayList<>(Arrays.asList(options)); + int i = 0; do { List subset = Randomly.subset(optionList); assertEquals(optionList.size(), 3); // check that the original set hasn't been modified @@ -49,7 +50,13 @@ public void testSubset() { } else { encounteredStrictSubsetNonEmpty = true; } - } while (!encounteredEmptySubset || !encounteredOriginalSet || !encounteredStrictSubsetNonEmpty); + } while (!encounteredEmptySubset || !encounteredOriginalSet || !encounteredStrictSubsetNonEmpty + || i++ < NR_MIN_RUNS); + + assertTrue(encounteredEmptySubset, "Empty subset was not encountered"); + assertTrue(encounteredOriginalSet, "Original set was not encountered"); + assertTrue(encounteredStrictSubsetNonEmpty, "Strict subset was not encountered"); + } @Test @@ -75,6 +82,10 @@ public void testString() { } } while (!encounteredInteger || !encounteredAscii || !encounteredNonAscii || !encounteredSpace || i++ < NR_MIN_RUNS); + assertTrue(encounteredInteger, "Integer was not encountered"); + assertTrue(encounteredAscii, "Ascii was not encountered"); + assertTrue(encounteredNonAscii, "Non ascii was not encountered"); + assertTrue(encounteredSpace, "Space was not encountered"); } @Test // TODO: also generate and check for NaN @@ -84,6 +95,7 @@ public void testDouble() { boolean encounteredPositive = false; boolean encounteredNegative = false; boolean encounteredInfinity = false; + int i = 0; do { double doubleVal = r.getDouble(); if (doubleVal == 0) { @@ -97,7 +109,12 @@ public void testDouble() { } else { fail(String.valueOf(doubleVal)); } - } while (!encounteredZero || !encounteredPositive || !encounteredNegative || !encounteredInfinity); + } while (!encounteredZero || !encounteredPositive || !encounteredNegative || !encounteredInfinity + || i++ < NR_MIN_RUNS); + assertTrue(encounteredZero, "Zero was not encountered"); + assertTrue(encounteredPositive, "Positive was not encountered"); + assertTrue(encounteredNegative, "Negative was not encountered"); + assertTrue(encounteredInfinity, "Infinity was not encountered"); } @Test @@ -123,6 +140,8 @@ public void testNonZeroInteger() { encounteredNegative = true; } } while (!encounteredPositive || !encounteredNegative || i++ < NR_MIN_RUNS); + assertTrue(encounteredPositive, "Positive integer was not encountered"); + assertTrue(encounteredNegative, "Negative integer was not encountered"); } @Test @@ -140,6 +159,8 @@ public void testPositiveInteger() { encounteredMaxValue = true; } } while (!encounteredZero || !encounteredMaxValue || i++ < NR_MIN_RUNS); + assertTrue(encounteredZero, "Zero was not encountered"); + assertTrue(encounteredMaxValue, "Max value was not encountered"); } @Test @@ -159,6 +180,9 @@ public void testBytes() { encounteredMax = true; } } while (!encounteredAllZeroes || !encounteredMax || !encounteredZeroLength || i++ < NR_MIN_RUNS); + assertTrue(encounteredAllZeroes, "All zeroes were not encountered"); + assertTrue(encounteredMax, "Max value was not encountered"); + assertTrue(encounteredZeroLength, "Zero length was not encountered"); } @Test From fb2c6b014abf2dd5206314fa14988aed23e04e3e Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Fri, 21 Mar 2025 12:38:18 +0000 Subject: [PATCH 647/856] Add unit tests for schema classes --- test/sqlancer/TestCommonSchema.java | 156 ++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 test/sqlancer/TestCommonSchema.java diff --git a/test/sqlancer/TestCommonSchema.java b/test/sqlancer/TestCommonSchema.java new file mode 100644 index 000000000..764e94ea4 --- /dev/null +++ b/test/sqlancer/TestCommonSchema.java @@ -0,0 +1,156 @@ +package sqlancer; + +import org.junit.jupiter.api.Test; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.TableIndex; + +import java.util.*; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.*; + +public class TestCommonSchema { + static class TestTable extends AbstractTable> { + TestTable(String name, List columns, List indexes, boolean isView) { + super(name, columns, indexes, isView); + } + + @Override + public long getNrRows(GlobalState globalState) { + return 0; + } + } + + static class TestTableColumn extends AbstractTableColumn { + TestTableColumn(String name, TestTable table, String type) { + super(name, table, type); + } + } + + static class TestSchema extends AbstractSchema, TestTable> { + TestSchema(List tables) { + super(tables); + } + } + + static class TestIndex extends TableIndex { + TestIndex(String name) { + super(name); + } + } + + private TestTable createTestTable(String name, List indexes, boolean isView, String... columns) { + List cols = Arrays.stream(columns) + .map(col -> new TestTableColumn(col, null, "VARCHAR")) + .collect(Collectors.toList()); + return new TestTable(name, cols, indexes, isView); + } + + private TestTableColumn createTestColumn(String name, TestTable table, String type) { + return new TestTableColumn(name, table, type); + } + + private TestSchema createTestSchema(TestTable... tables) { + return new TestSchema(Arrays.asList(tables)); + } + + @Test + void testColumnManagement() { + TestTable table = createTestTable("products", Collections.emptyList(), false, "sku", "price"); + TestTableColumn randomCol = table.getRandomColumn(); + + assertEquals("sku", table.getColumns().get(0).getName()); + assertEquals("VARCHAR", table.getColumns().get(1).getType()); + + assertTrue(table.getColumns().contains(randomCol)); + } + + @Test + void testIndexManagement() { + TestIndex idx1 = new TestIndex("idx_sku"); + TestIndex idx2 = new TestIndex("idx_price"); + TestTable table = createTestTable("products", Arrays.asList(idx1, idx2), false, "sku", "price"); + TableIndex randomIndex = table.getRandomIndex(); + + assertTrue(table.hasIndexes()); + assertEquals(2, table.getIndexes().size()); + assertTrue(table.getIndexes().contains(randomIndex)); + } + + @Test + void testViewManagement() { + TestTable view1 = createTestTable("v1", Collections.emptyList(),true, "col1"); + TestTable view2 = createTestTable("v2", Collections.emptyList(),true, "col2"); + TestTable table = createTestTable("t1", Collections.emptyList(), false, "col3"); + TestSchema schema = createTestSchema(view1, view2, table); + + assertAll( + () -> assertEquals(2, schema.getViews().size(), "Should detect 2 views"), + () -> assertEquals(1, schema.getDatabaseTablesWithoutViews().size(), "Should detect 1 normal table"), + () -> assertEquals("t1", schema.getDatabaseTablesWithoutViews().get(0).getName()) + ); + } + + @Test + void testFreeColumnNameGeneration() { + TestTable table = createTestTable("users", Collections.emptyList(), false, "id", "name"); + Set generatedNames = new HashSet<>(); + + for (int i = 0; i < 100; i++) { + String newName = table.getFreeColumnName(); + assertTrue(generatedNames.add(newName), "Duplicate: " + newName); + + List newColumns = new ArrayList<>(table.getColumns()); + newColumns.add(new TestTableColumn(newName, table, "TEXT")); + table = new TestTable(table.getName(), newColumns, table.getIndexes(), table.isView()); + } + } + + @Test + void testObjectComparison() { + TestTable tableA = createTestTable("A", Collections.emptyList(), false, "x", "y"); + TestTable tableB = createTestTable("B", Collections.emptyList(), false, "b"); + + TestTableColumn colA1 = new TestTableColumn("x", tableA, "INT"); + TestTableColumn colA2 = new TestTableColumn("y", tableA, "INT"); + TestTableColumn colB1 = new TestTableColumn("b", tableB, "TEXT"); + + assertAll( + () -> assertTrue(colA1.compareTo(colA2) < 0, "Columns should be ordered by name"), + () -> assertTrue(colA1.compareTo(colB1) < 0, "Columns should be ordered by name"), + () -> assertTrue(tableA.compareTo(tableB) > 0, "Tables should be ordered reverse-alphabetically"), + () -> assertEquals(0, tableA.compareTo(tableA), "Same table should be equal") + ); + } + + @Test + void testEquality() { + TestTable table1 = createTestTable("t1", Collections.emptyList(), false, "id"); + TestTable table2 = createTestTable("t2", Collections.emptyList(), false, "id"); + + TestTableColumn col1 = new TestTableColumn("id", table1, "INT"); + TestTableColumn col2 = new TestTableColumn("id", table1, "INT"); + TestTableColumn col3 = new TestTableColumn("id", table2, "INT"); + TestTableColumn col4 = new TestTableColumn("name", table1, "TEXT"); + + assertAll( + () -> assertEquals(col1, col2, "Same table/column should be equal"), + () -> assertNotEquals(col1, col3, "Different tables should not be equal"), + () -> assertNotEquals(col1, col4, "Different columns should not be equal"), + () -> assertNotEquals(col1, "invalid_object", "Different types should not be equal") + ); + } + + @Test + void testBoundaryConditions() { + String longName = "a".repeat(256); + TestTableColumn longCol = new TestTableColumn(longName, null, "TEXT"); + assertEquals(longName, longCol.getName()); + + TestTableColumn col2 = createTestColumn("orphan", null, "UNKNOWN"); + assertEquals("orphan", col2.getFullQualifiedName()); + assertNull(col2.getTable()); + } +} From 17952a41abe734da4e401faad501e626811c9158 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Fri, 3 Jan 2025 20:02:31 +0800 Subject: [PATCH 648/856] add the implementation of CODDTest for SQLite3 --- src/sqlancer/common/oracle/CODDTestBase.java | 25 + .../common/schema/AbstractTables.java | 24 + .../sqlite3/SQLite3ExpectedValueVisitor.java | 46 + src/sqlancer/sqlite3/SQLite3Options.java | 3 + .../sqlite3/SQLite3OracleFactory.java | 12 + .../sqlite3/SQLite3ToStringVisitor.java | 158 ++- src/sqlancer/sqlite3/SQLite3Visitor.java | 36 + .../sqlite3/ast/SQLite3Expression.java | 201 +++- src/sqlancer/sqlite3/ast/SQLite3Select.java | 32 + .../sqlite3/oracle/SQLite3CODDTestOracle.java | 1002 +++++++++++++++++ .../sqlite3/schema/SQLite3DataType.java | 17 + 11 files changed, 1552 insertions(+), 4 deletions(-) create mode 100644 src/sqlancer/common/oracle/CODDTestBase.java create mode 100644 src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java diff --git a/src/sqlancer/common/oracle/CODDTestBase.java b/src/sqlancer/common/oracle/CODDTestBase.java new file mode 100644 index 000000000..162a8eaed --- /dev/null +++ b/src/sqlancer/common/oracle/CODDTestBase.java @@ -0,0 +1,25 @@ +package sqlancer.common.oracle; + +import sqlancer.Main.StateLogger; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; + +public abstract class CODDTestBase> implements TestOracle { + protected final S state; + protected final ExpectedErrors errors = new ExpectedErrors(); + protected final StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + protected String auxiliaryQueryString; + protected String foldedQueryString; + protected String originalQueryString; + + public CODDTestBase(S state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } +} \ No newline at end of file diff --git a/src/sqlancer/common/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java index 2afff82ab..c75cb30e1 100644 --- a/src/sqlancer/common/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -34,4 +34,28 @@ public String columnNamesAsString(Function function) { return getColumns().stream().map(function).collect(Collectors.joining(", ")); } + + public void addTable(T table) { + if (!this.tables.contains(table)) { + this.tables.add(table); + columns.addAll(table.getColumns()); + } + } + + public void removeTable(T table) { + if (this.tables.contains(table)) { + this.tables.remove(table); + for (C c : table.getColumns()) { + columns.remove(c); + } + } + } + + public Boolean isContained(T table) { + return this.tables.contains(table); + } + + public int getSize() { + return this.tables.size(); + } } diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 1c148ee86..40f8ae20a 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -14,14 +14,21 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -303,4 +310,43 @@ public void visit(SQLite3SetClause set) { visit(set.getRight()); } + @Override + public void visit(SQLite3Alias alias) { + print(alias); + print(alias.getOrigonalExpression()); + print(alias.getAliasExpression()); + } + + @Override + public void visit(SQLite3WithClasure withClasure) { + print(withClasure); + print(withClasure.getLeft()); + print(withClasure.getRight()); + } + + @Override + public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { + print(tableAndColumnRef); + } + + @Override + public void visit(SQLite3Values values) { + print(values); + } + + @Override + public void visit(SQLite3ExpressionBag expr) { + print(expr); + print(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3Typeof expr) { + print(expr); + print(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3ResultMap tableSummary) { + } } diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index ec90f3bca..47a80e832 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -79,6 +79,9 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") public int maxNumIndexes = 20; + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") + public String coddTestModel = "random"; + @Override public List getTestOracleFactory() { return Arrays.asList(oracles); diff --git a/src/sqlancer/sqlite3/SQLite3OracleFactory.java b/src/sqlancer/sqlite3/SQLite3OracleFactory.java index b1a9a1265..2e2f9f0f7 100644 --- a/src/sqlancer/sqlite3/SQLite3OracleFactory.java +++ b/src/sqlancer/sqlite3/SQLite3OracleFactory.java @@ -11,6 +11,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.oracle.SQLite3CODDTestOracle; import sqlancer.sqlite3.oracle.SQLite3Fuzzer; import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; import sqlancer.sqlite3.oracle.tlp.SQLite3TLPAggregateOracle; @@ -96,6 +97,17 @@ public TestOracle create(SQLite3GlobalState globalState) thr oracles.add(AGGREGATE.create(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + CODDTest { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3CODDTestOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } }; } diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 09eb224fb..44aa0d1e0 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -1,7 +1,11 @@ package sqlancer.sqlite3; import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; @@ -11,6 +15,7 @@ import sqlancer.sqlite3.ast.SQLite3Case.SQLite3CaseWithoutBaseExpression; import sqlancer.sqlite3.ast.SQLite3Cast; import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.ast.SQLite3Constant.SQLite3NullConstant; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.BetweenOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Cast; @@ -19,12 +24,19 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; @@ -35,6 +47,7 @@ import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecBetween; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecTerm; +import sqlancer.sqlite3.schema.SQLite3DataType; public class SQLite3ToStringVisitor extends ToStringVisitor implements SQLite3Visitor { @@ -99,6 +112,10 @@ public void visit(SQLite3Select s, boolean inner) { if (inner) { sb.append("("); } + if (s.getWithClasure() != null) { + visit(s.getWithClasure()); + sb.append(" "); + } sb.append("SELECT "); switch (s.getFromOptions()) { case DISTINCT: @@ -288,13 +305,20 @@ public void visit(InOperation op) { sb.append("("); visit(op.getLeft()); sb.append(" IN "); - sb.append("("); if (op.getRightExpressionList() != null) { + sb.append("("); visit(op.getRightExpressionList()); + sb.append(")"); } else { - visit(op.getRightSelect()); + if (op.getRightSelect() instanceof SQLite3Expression.SQLite3TableReference) { + visit(op.getRightSelect()); + } else { + sb.append("("); + visit(op.getRightSelect()); + sb.append(")"); + } } - sb.append(")"); + sb.append(")"); } @@ -305,6 +329,9 @@ public void visit(Subquery query) { @Override public void visit(SQLite3Exist exist) { + if (exist.getNegated()) { + sb.append(" NOT"); + } sb.append(" EXISTS "); if (exist.getExpression() instanceof SQLite3SetClause) { sb.append("("); @@ -482,4 +509,129 @@ public void visit(SQLite3SetClause set) { sb.append(SQLite3Visitor.asString(set.getRight())); } + @Override + public void visit(SQLite3Alias alias) { + sb.append("("); + visit(alias.getOrigonalExpression()); + sb.append(")"); + sb.append(" AS "); + visit(alias.getAliasExpression()); + } + + @Override + public void visit(SQLite3WithClasure withClasure) { + sb.append("WITH "); + visit(withClasure.getLeft()); + sb.append(" AS "); + visit(withClasure.getRight()); + } + + @Override + public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { + sb.append(tableAndColumnRef.getString()); + } + + @Override + public void visit(SQLite3Values values) { + Map> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + sb.append("(VALUES "); + for (int i = 0; i < size; i++) { + sb.append("("); + for (String name : columnNames) { + if (vs.get(name).get(i).getDataType() == SQLite3DataType.NULL) { + visit(vs.get(name).get(i)); + } else { + sb.append("(CAST("); + visit(vs.get(name).get(i)); + sb.append(" AS "); + switch(vs.get(name).get(i).getDataType()) { + case BINARY: + sb.append("BLOB))"); + break; + case INT: + sb.append("INT))"); + break; + case TEXT: + sb.append("TEXT))"); + break; + case REAL: + sb.append("REAL))"); + break; + default: + throw new IgnoreMeException(); + } + } + + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("), "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + } + + @Override + public void visit(SQLite3ExpressionBag expr) { + visit(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3Typeof expr) { + sb.append("typeof("); + visit(expr.getInnerExpr()); + sb.append(")"); + } + + @Override + public void visit(SQLite3ResultMap tableSummary) { + // we utlize CASE WHEN THEN END here + SQLite3Values values = tableSummary.getValues(); + List columnRefs = tableSummary.getColumns(); + List summary = tableSummary.getSummary(); + + Map> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + if (size == 0) { + sb.append("("); + for (int j = 0; j < columnRefs.size(); ++j) { + visit(columnRefs.get(j)); + sb.append(" IS NULL AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return; + } + List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + sb.append(" CASE "); + for (int i = 0; i < size; i++) { + sb.append("WHEN "); + for (int j = 0; j < columnNames.size(); ++j) { + visit(columnRefs.get(j)); + if (vs.get(columnNames.get(j)).get(i) instanceof SQLite3NullConstant) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(vs.get(columnNames.get(j)).get(i).toString()); + } + sb.append(" AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("THEN "); + visit(summary.get(i)); + sb.append(" "); + } + sb.append("END "); + } } diff --git a/src/sqlancer/sqlite3/SQLite3Visitor.java b/src/sqlancer/sqlite3/SQLite3Visitor.java index f891c17f0..d5ebce096 100644 --- a/src/sqlancer/sqlite3/SQLite3Visitor.java +++ b/src/sqlancer/sqlite3/SQLite3Visitor.java @@ -13,14 +13,21 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -130,6 +137,21 @@ default void visit(SQLite3PostfixUnaryOperation exp) { void visit(SQLite3WindowFunctionFrameSpecBetween between); + + void visit(SQLite3Alias alias); + + void visit(SQLite3WithClasure withClasure); + + void visit(SQLite3TableAndColumnRef tableAndColumnRef); + + void visit(SQLite3Values values); + + void visit(SQLite3ExpressionBag expr); + + void visit(SQLite3Typeof expr); + + void visit(SQLite3ResultMap tableSummary); + default void visit(SQLite3Expression expr) { if (expr instanceof Sqlite3BinaryOperation) { visit((Sqlite3BinaryOperation) expr); @@ -193,6 +215,20 @@ default void visit(SQLite3Expression expr) { visit((SQLite3TableReference) expr); } else if (expr instanceof SQLite3SetClause) { visit((SQLite3SetClause) expr); + } else if (expr instanceof SQLite3Alias) { + visit((SQLite3Alias) expr); + } else if (expr instanceof SQLite3WithClasure) { + visit((SQLite3WithClasure) expr); + } else if (expr instanceof SQLite3TableAndColumnRef) { + visit((SQLite3TableAndColumnRef) expr); + } else if (expr instanceof SQLite3Values) { + visit((SQLite3Values) expr); + } else if (expr instanceof SQLite3ExpressionBag) { + visit((SQLite3ExpressionBag) expr); + } else if (expr instanceof SQLite3Typeof) { + visit((SQLite3Typeof) expr); + } else if (expr instanceof SQLite3ResultMap) { + visit((SQLite3ResultMap) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 7b131b6b5..a14432eec 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -2,6 +2,7 @@ import java.util.List; import java.util.Optional; +import java.util.Map; import sqlancer.IgnoreMeException; import sqlancer.LikeImplementationHelper; @@ -113,9 +114,19 @@ public SQLite3CollateSequence getImplicitCollateSequence() { public static class SQLite3Exist extends SQLite3Expression { private final SQLite3Expression select; + private boolean negated = false; - public SQLite3Exist(SQLite3Expression select) { + public SQLite3Exist(SQLite3Expression select, boolean negated) { this.select = select; + this.negated = negated; + } + + public void setNegated(boolean negated) { + this.negated = negated; + } + + public boolean getNegated() { + return this.negated; } public SQLite3Expression getExpression() { @@ -1552,4 +1563,192 @@ public boolean omitBracketsWhenPrinting() { } } + public static class SQLite3WithClasure extends SQLite3Expression { + + private SQLite3Expression left; + private SQLite3Expression right; + + public SQLite3WithClasure(SQLite3Expression left, SQLite3Expression right) { + this.left = left; + this.right = right; + } + + public SQLite3Expression getLeft() { + return this.left; + } + + public SQLite3Expression getRight() { + return this.right; + } + + public void updateRight(SQLite3Expression right) { + this.right = right; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3Alias extends SQLite3Expression { + + private SQLite3Expression origonalExpression; + private SQLite3Expression aliasExpression; + + public SQLite3Alias(SQLite3Expression origonalExpression, SQLite3Expression aliasExpression) { + this.origonalExpression = origonalExpression; + this.aliasExpression = aliasExpression; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + public SQLite3Expression getOrigonalExpression() { + return origonalExpression; + } + + public SQLite3Expression getAliasExpression() { + return aliasExpression; + } + } + + public static class SQLite3TableAndColumnRef extends SQLite3Expression { + + private final SQLite3Table table; + + public SQLite3TableAndColumnRef(SQLite3Table table) { + this.table = table; + } + + public SQLite3Table getTable() { + return this.table; + } + + public String getString() { + StringBuilder sb = new StringBuilder(); + sb.append(table.getName()); + sb.append("("); + for (SQLite3Column c : this.table.getColumns()) { + sb.append(c.getName()); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3Values extends SQLite3Expression { + + private final Map> values; + private final List columns; + + public SQLite3Values(Map> values, List columns) { + this.values = values; + this.columns = columns; + } + + public Map> getValues() { + return this.values; + } + + public List getColumns() { + return this.columns; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3ExpressionBag extends SQLite3Expression { + private SQLite3Expression innerExpr; + + public SQLite3ExpressionBag(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public void updateInnerExpr(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public SQLite3Expression getInnerExpr() { + return innerExpr; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } + + public static class SQLite3Typeof extends SQLite3Expression { + private SQLite3Expression innerExpr; + + public SQLite3Typeof(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public SQLite3Expression getInnerExpr() { + return innerExpr; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } + + public static class SQLite3ResultMap extends SQLite3Expression { + private final SQLite3Values values; + private final List columns; + private final List summary; + private final SQLite3DataType summaryDataType; + + public SQLite3ResultMap(SQLite3Values values, List columns, List summary, SQLite3DataType summaryDataType) { + this.values = values; + this.columns = columns; + this.summary = summary; + this.summaryDataType = summaryDataType; + + Map> vs = values.getValues(); + if (vs.get(vs.keySet().iterator().next()).size() != summary.size()) { + throw new AssertionError(); + } + } + + public SQLite3Values getValues() { + return this.values; + } + + public List getColumns() { + return this.columns; + } + + public List getSummary() { + return this.summary; + } + + public SQLite3DataType getSummaryDataType() { + return this.summaryDataType; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 57eb91b0b..042aab856 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -4,6 +4,7 @@ import java.util.Collections; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.common.ast.newast.Select; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression.Join; @@ -24,6 +25,7 @@ public class SQLite3Select extends SQLite3Expression private List fetchColumns = Collections.emptyList(); private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; + private SQLite3WithClasure withClause = null; public SQLite3Select() { } @@ -42,6 +44,7 @@ public SQLite3Select(SQLite3Select other) { joinStatements.add(new Join(j)); } havingClause = other.havingClause; + withClause = other.withClause; } public enum SelectType { @@ -161,4 +164,33 @@ public SQLite3Expression getHavingClause() { public String asString() { return SQLite3Visitor.asString(this); } + + public void setWithClasure(SQLite3WithClasure withClasure) { + this.withClause = withClasure; + } + + public void updateWithClasureRight(SQLite3Expression withClasureRight) { + this.withClause.updateRight(withClasureRight); + } + + public SQLite3Expression getWithClasure() { + return this.withClause; + } + + public void replaceFromTable(String tableName, SQLite3Expression newFromExpression) { + int replaceIdx = -1; + for (int i = 0; i < fromList.size(); ++i) { + SQLite3Expression f = fromList.get(i); + if (f instanceof SQLite3TableReference) { + SQLite3TableReference tableRef = (SQLite3TableReference) f; + if (tableRef.getTable().getName() == tableName) { + replaceIdx = i; + } + } + } + if (replaceIdx == -1) { + throw new IgnoreMeException(); + } + fromList.set(replaceIdx, newFromExpression); + } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java new file mode 100644 index 000000000..d20e6e9be --- /dev/null +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -0,0 +1,1002 @@ +package sqlancer.sqlite3.oracle; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.CODDTestBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3Provider; +import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Aggregate; +import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.ast.SQLite3Constant.SQLite3TextConstant; +import sqlancer.sqlite3.ast.SQLite3Expression; +import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; +import sqlancer.sqlite3.ast.SQLite3Select; +import sqlancer.sqlite3.gen.SQLite3Common; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.schema.SQLite3DataType; +import sqlancer.sqlite3.schema.SQLite3Schema; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; + +public class SQLite3CODDTestOracle extends CODDTestBase implements TestOracle { + + private final SQLite3Schema s; + private SQLite3ExpressionGenerator gen; + private Reproducer reproducer; + + private String tempTableName = "temp_table"; + + private SQLite3Expression foldedExpr; + private SQLite3Expression constantResOfFoldedExpr; + + private List tablesFromOuterContext = new ArrayList<>(); + private List joinsInExpr = null; + + Map> auxiliaryQueryResult = new HashMap<>(); + Map> selectResult = new HashMap<>(); + + Boolean useSubqueryAsFoldedExpr; + Boolean useCorrelatedSubqueryAsFoldedExpr; + + public SQLite3CODDTestOracle(SQLite3GlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + SQLite3Errors.addExpectedExpressionErrors(errors); + SQLite3Errors.addMatchQueryErrors(errors); + SQLite3Errors.addQueryErrors(errors); + errors.add("misuse of aggregate"); + errors.add("misuse of window function"); + errors.add("second argument to nth_value must be a positive integer"); + errors.add("no such table"); + errors.add("no query solution"); + errors.add("unable to use function MATCH in the requested context"); + errors.add("[SQLITE_ERROR] SQL error or missing database (unrecognized token:"); + } + + @Override + public void check() throws SQLException { + reproducer = null; + + useSubqueryAsFoldedExpr = useSubquery(); + useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); + + SQLite3Select auxiliaryQuery = null; + if (useSubqueryAsFoldedExpr) { + if (useCorrelatedSubqueryAsFoldedExpr) { + auxiliaryQuery = genSelectWithCorrelatedSubquery(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + + auxiliaryQueryResult.putAll(selectResult); + } else { + auxiliaryQuery = genSelectExpression(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); + } + } else { + auxiliaryQuery = genSimpleSelect(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + + auxiliaryQueryResult.putAll(selectResult); + } + + + SQLite3Select originalQuery = null; + + Map> foldedResult = new HashMap<>(); + Map> originalResult = new HashMap<>(); + + // dependent expression + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + // original query + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(this.foldedExpr); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // independent expression + // empty result, put the inner query in (NOT) EXIST + else if (auxiliaryQueryResult.size() == 0 || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).size() == 0) { + boolean isNegated = Randomly.getBoolean() ? false : true; + // original query + SQLite3Exist existExpr = new SQLite3Exist(new SQLite3Select(auxiliaryQuery), isNegated); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(existExpr); + + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + SQLite3Expression equivalentExpr = isNegated ? SQLite3Constant.createTrue() : SQLite3Constant.createFalse(); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant + else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 && Randomly.getBoolean()) { + // original query + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(auxiliaryQuery); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + SQLite3Expression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); + specificCondition.updateInnerExpr(equivalentExpr);; + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // one column + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability()) { + // else if (auxiliaryQueryResult.size() == 1 && false) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + SQLite3ColumnName selectedColumn = new SQLite3ColumnName(Randomly.fromList(columns), null); + SQLite3Table selectedTable = selectedColumn.getColumn().getTable(); + InOperation INOperation = new InOperation(selectedColumn, new SQLite3Select(auxiliaryQuery)); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(INOperation); + + originalQuery = this.genSelectExpression(selectedTable, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + // folded query + // can not use IN VALUES here, because there is no affinity for the right operand of IN when right operand is a list + try { + SQLite3Table t = this.createTemporaryTable(auxiliaryQuery, "intable"); + SQLite3TableReference equivalentTable = new SQLite3TableReference(t); + INOperation = new InOperation(selectedColumn, equivalentTable); + specificCondition.updateInnerExpr(INOperation); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable("intable"); + } + } + // There is not `ANY` and `ALL` operator in SQLite3 + // Row Subquery + else { + // original query + SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); + originalQuery = this.genSelectExpression(temporaryTable, null); + SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); + SQLite3WithClasure withClasure = new SQLite3WithClasure(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); + originalQuery.setWithClasure(withClasure); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + // folded query + if (Randomly.getBoolean() && this.testCommonTableExpression()) { + // there are too many false positives + // common table expression + // folded query: WITH table AS VALUES () + SQLite3Values values = new SQLite3Values(auxiliaryQueryResult, temporaryTable.getColumns()); + originalQuery.updateWithClasureRight(values); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (Randomly.getBoolean() && this.testDerivedTable()) { + // derived table + // folded query: SELECT FROM () AS table + originalQuery.setWithClasure(null); + SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); + SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); + originalQuery.replaceFromTable(this.tempTableName, alias); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (this.testInsert()){ + // there are too many false positives + // folded query: CREATE the table and INSERT INTO table subquery + try { + this.createTemporaryTable(auxiliaryQuery, this.tempTableName); + originalQuery.setWithClasure(null); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable(this.tempTableName); + } + } else { + throw new IgnoreMeException(); + } + } + if (foldedResult == null || originalResult == null) { + throw new IgnoreMeException(); + } + if (foldedQueryString.equals(originalQueryString)) { + throw new IgnoreMeException(); + } + if (!compareResult(foldedResult, originalResult)) { + reproducer = null; // TODO + state.getState().getLocalState().log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError(auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); + } + } + + private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Expression specificCondition) { + SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); + if (tempTable != null) { + randomTables.addTable(tempTable); + } + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + for (SQLite3Table t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + randomTables.removeTable(t); + } + } + } + + List columns = randomTables.getColumns(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + List joinStatements = new ArrayList<>(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + joinStatements.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } + else if (Randomly.getBoolean()) { + joinStatements = genJoinExpression(gen, tables, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); + } + List tableRefs = SQLite3Common.getTableRefs(tables, s); + SQLite3Select select = new SQLite3Select(); + select.setFromList(tableRefs); + if (joinStatements.size() > 0) { + select.setJoinClauses(joinStatements); + } + + SQLite3Expression randomWhereCondition = gen.generateExpression(); + SQLite3Expression whereCondition = null; + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomWhereCondition, specificCondition, operator); + } else { + whereCondition = randomWhereCondition; + } + select.setWhereClause(whereCondition); + + if (Randomly.getBoolean()) { + select.setOrderByClauses(genOrderBysExpression(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + } + + if (Randomly.getBoolean()) { + List selectedColumns = Randomly.nonEmptySubset(columns); + List selectedAlias = new LinkedList<>(); + for (int i = 0; i < selectedColumns.size(); ++i) { + SQLite3ColumnName originalName = new SQLite3ColumnName(selectedColumns.get(i), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(i)), null); + SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); + selectedAlias.add(columnAlias); + } + select.setFetchColumns(selectedAlias); + } else { + SQLite3ColumnName aggr = new SQLite3ColumnName(Randomly.fromList(columns), null); + SQLite3Provider.mustKnowResult = true; + SQLite3Expression originalName = new SQLite3Aggregate(Arrays.asList(aggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c0"), null); + SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); + select.setFetchColumns(Arrays.asList(columnAlias)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List groupByClause = genGroupByClause(columns, specificCondition); + select.setGroupByClause(groupByClause); + if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + select.setHavingClause(genHavingClause(columns, specificCondition)); + } + } + } + return select; + } + + // For expression test + private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression specificCondition) { + SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); + if (tempTable != null) { + randomTables.addTable(tempTable); + } + if (!useSubqueryAsFoldedExpr) { + for (SQLite3Table t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + randomTables.removeTable(t); + } + } + } + + List columns = randomTables.getColumns(); + if (!useSubqueryAsFoldedExpr && this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + tablesFromOuterContext = randomTables.getTables(); + + if (joinsInExpr == null) { + if (Randomly.getBooleanWithRatherLowProbability()) { + joinsInExpr = genJoinExpression(gen, tables, null, true); + } else { + joinsInExpr = new ArrayList(); + } + } + + List tableRefs = SQLite3Common.getTableRefs(tables, s); + SQLite3Select select = new SQLite3Select(); + select.setFromList(tableRefs); + if (joinsInExpr != null && joinsInExpr.size() > 0) { + select.setJoinClauses(joinsInExpr); + } + + SQLite3Expression whereCondition = gen.generateExpression(); + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(whereCondition, specificCondition, operator); + } + this.foldedExpr = whereCondition; + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (SQLite3Column c : randomTables.getColumns()) { + SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); + fetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(whereCondition, aliasName); + fetchColumns.add(columnAlias); + + select.setFetchColumns(fetchColumns); + + originalQueryString = SQLite3Visitor.asString(select); + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the summary from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + List tempColumnList = new ArrayList<>(); + + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + SQLite3Alias cAlias = (SQLite3Alias) fetchColumns.get(i); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3Column column = cRef.getColumn(); + String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); + SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); + tempColumnList.add(newColumn); + } + List columnRef = new ArrayList<>(); + for (SQLite3Column c : randomTables.getColumns()) { + columnRef.add(new SQLite3ColumnName(c, null)); + } + if (tempColumnList.size() != queryRes.size()) { + throw new AssertionError(); + } + SQLite3Values values = new SQLite3Values(queryRes, tempColumnList); + this.constantResOfFoldedExpr = new SQLite3ResultMap(values, columnRef, summary, null); + + return select; + } + + private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable, SQLite3Expression specificCondition) { + // do not support join now + this.joinsInExpr = new ArrayList(); + + SQLite3Tables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); + SQLite3Tables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); + + if (selectedTable != null) { + innerQueryRandomTables.addTable(selectedTable); + } + + List innerQueryFromTables = new ArrayList<>(); + for (SQLite3Table t : innerQueryRandomTables.getTables()) { + if (!outerQueryRandomTables.isContained(t)) { + innerQueryFromTables.add(new SQLite3TableReference(t)); + } + } + for (SQLite3Table t : outerQueryRandomTables.getTables()) { + if (innerQueryRandomTables.isContained(t)) { + innerQueryRandomTables.removeTable(t); + + List newColumns = new ArrayList<>(); + for (SQLite3Column c : t.getColumns()) { + SQLite3Column newColumn = new SQLite3Column(c.getName(), c.getType(), false, null, false); + newColumns.add(newColumn); + } + SQLite3Table newTable = new SQLite3Table(t.getName() + "a", newColumns, null, true, false, false, false); + for (SQLite3Column c : newColumns) { + c.setTable(newTable); + } + innerQueryRandomTables.addTable(newTable); + + SQLite3Alias alias = new SQLite3Alias(new SQLite3TableReference(t), new SQLite3TableReference(newTable)); + innerQueryFromTables.add(alias); + } + } + + List innerQueryColumns = new ArrayList<>(); + innerQueryColumns.addAll(innerQueryRandomTables.getColumns()); + innerQueryColumns.addAll(outerQueryRandomTables.getColumns()); + gen = new SQLite3ExpressionGenerator(state).setColumns(innerQueryColumns); + + SQLite3Select innerQuery = new SQLite3Select(); + innerQuery.setFromList(innerQueryFromTables); + + SQLite3Expression innerQueryWhereCondition = gen.generateExpression(); + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + innerQueryWhereCondition = new SQLite3Expression.Sqlite3BinaryOperation(innerQueryWhereCondition, specificCondition, operator); + } + innerQuery.setWhereClause(innerQueryWhereCondition); + + // use aggregate function in fetch column + SQLite3ColumnName innerQueryAggr = new SQLite3ColumnName(Randomly.fromList(innerQueryRandomTables.getColumns()), null); + SQLite3Provider.mustKnowResult = true; + SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List groupByClause = genGroupByClause(innerQueryColumns, specificCondition); + innerQuery.setGroupByClause(groupByClause); + if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + innerQuery.setHavingClause(genHavingClause(innerQueryColumns, specificCondition)); + } + } + + this.foldedExpr = innerQuery; + + + // outer query + SQLite3Select outerQuery = new SQLite3Select(); + outerQuery.setFromList(SQLite3Common.getTableRefs(outerQueryRandomTables.getTables(), s)); + tablesFromOuterContext = outerQueryRandomTables.getTables(); + + List outerQueryFetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (SQLite3Column c : outerQueryRandomTables.getColumns()) { + SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); + outerQueryFetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(innerQuery, aliasName); + outerQueryFetchColumns.add(columnAlias); + + outerQuery.setFetchColumns(outerQueryFetchColumns); + + originalQueryString = SQLite3Visitor.asString(outerQuery); + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the summary from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + List tempColumnList = new ArrayList<>(); + + for (int i = 0; i < outerQueryFetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + SQLite3Alias cAlias = (SQLite3Alias) outerQueryFetchColumns.get(i); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3Column column = cRef.getColumn(); + String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); + SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); + tempColumnList.add(newColumn); + } + List columnRef = new ArrayList<>(); + for (SQLite3Column c : outerQueryRandomTables.getColumns()) { + columnRef.add(new SQLite3ColumnName(c, null)); + } + if (tempColumnList.size() != queryRes.size()) { + throw new AssertionError(); + } + SQLite3Values values = new SQLite3Values(queryRes, tempColumnList); + this.constantResOfFoldedExpr = new SQLite3ResultMap(values, columnRef, summary, null); + + return outerQuery; + } + + private List genJoinExpression(SQLite3ExpressionGenerator gen, List tables, SQLite3Expression specificCondition, boolean joinForExperssion) { + List joinStatements = new ArrayList<>(); + if (!state.getDbmsSpecificOptions().testJoins) { + return joinStatements; + } + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + if (Randomly.getBoolean() && tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1 || joinForExperssion) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + SQLite3Expression randomOnCondition = gen.generateExpression(); + SQLite3Expression onCondition = null; + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + onCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomOnCondition, specificCondition, operator); + } else { + onCondition = randomOnCondition; + } + + SQLite3Table table = Randomly.fromList(tables); + tables.remove(table); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + onCondition = null; + } + Join j = new SQLite3Expression.Join(table, onCondition, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } + + private List genOrderBysExpression(SQLite3ExpressionGenerator gen, SQLite3Expression specificCondition) { + List expressions = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + expressions.add(genOrderingTerm(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + } + return expressions; + } + + private SQLite3Expression genOrderingTerm(SQLite3ExpressionGenerator gen, SQLite3Expression specificCondition) { + SQLite3Expression expr = gen.generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + // COLLATE is potentially already generated + if (Randomly.getBoolean()) { + expr = new SQLite3OrderingTerm(expr, Ordering.getRandomValue()); + } + if (state.getDbmsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { + expr = new SQLite3PostfixText(expr, Randomly.fromOptions(" NULLS FIRST", " NULLS LAST"), + null /* expr.getExpectedValue() */) { + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + }; + } + return expr; + } + + private List genGroupByClause(List columns, SQLite3Expression specificCondition) { + errors.add("GROUP BY term out of range"); + if (Randomly.getBoolean()) { + List collect = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + SQLite3Expression expr = new SQLite3ExpressionGenerator(state).setColumns(columns).generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + collect.add(expr); + } + return collect; + } + return Collections.emptyList(); + } + + private SQLite3Expression genHavingClause(List columns, SQLite3Expression specificCondition) { + SQLite3Expression expr = new SQLite3ExpressionGenerator(state).setColumns(columns).generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + return expr; + } + + private Map> getQueryResult(String queryString, SQLite3GlobalState state) throws SQLException { + Map> result = new LinkedHashMap<>(); + if (options.logEachSelect()) { + logger.writeCurrentNoLineBreak(queryString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + stmt.setQueryTimeout(600); + ResultSet rs = null; + try { + rs = stmt.executeQuery(queryString); + ResultSetMetaData metaData = rs.getMetaData(); + Integer columnCount = metaData.getColumnCount(); + Map idxNameMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + result.put("c" + String.valueOf(i-1), new ArrayList<>()); + idxNameMap.put(i, "c" + String.valueOf(i-1)); + } + + int resultRows = 0; + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + try { + Object value = rs.getObject(i); + SQLite3Constant constant; + if (rs.wasNull()) { + constant = SQLite3Constant.createNullConstant(); + } + + else if (value instanceof Integer) { + constant = SQLite3Constant.createIntConstant(Long.valueOf((Integer) value)); + } else if (value instanceof Short) { + constant = SQLite3Constant.createIntConstant(Long.valueOf((Short) value)); + } else if (value instanceof Long) { + constant = SQLite3Constant.createIntConstant((Long) value); + } + + else if (value instanceof Double) { + constant = SQLite3Constant.createRealConstant((double) value); + } else if (value instanceof Float) { + constant = SQLite3Constant.createRealConstant(((Float) value).doubleValue()); + } else if (value instanceof BigDecimal) { + constant = SQLite3Constant.createRealConstant(((BigDecimal) value).doubleValue()); + } + + else if (value instanceof Byte) { + constant = SQLite3Constant.createBinaryConstant((byte[]) value); + } else if (value instanceof byte[]) { + constant = SQLite3Constant.createBinaryConstant((byte[]) value); + } else if (value instanceof Boolean) { + constant = SQLite3Constant.createBoolean((boolean) value); + } else if (value instanceof String) { + constant = SQLite3Constant.createTextConstant((String) value); + } else if (value == null) { + constant = SQLite3Constant.createNullConstant(); + } else { + throw new IgnoreMeException(); + } + List v = result.get(idxNameMap.get(i)); + v.add(constant); + } catch (SQLException e) { + System.out.println(e.getMessage()); + throw new IgnoreMeException(); + } + } + ++resultRows; + if (resultRows > 100) { + throw new IgnoreMeException(); + } + } + Main.nrSuccessfulActions.addAndGet(1); + rs.close(); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + if (rs != null) { + rs.close(); + } + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + return result; + } + + private SQLite3Table genTemporaryTable(SQLite3Select select, String tableName) { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); + databaseColumns.add(column); + } + SQLite3Table table = new SQLite3Table(tableName, databaseColumns, null, false, false, false, false); + for (SQLite3Column c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName) throws SQLException { + String selectString = SQLite3Visitor.asString(select); + Map idxTypeMap = getColumnTypeFromSelect(select); + + Integer columnNumber = idxTypeMap.size(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE " + tableName + " ("); + for (int i = 0; i < columnNumber; ++i) { + String columnTypeName = ""; + if (idxTypeMap.get(i) != null) { + switch (idxTypeMap.get(i)) { + case INT: + case TEXT: + case REAL: + columnTypeName = idxTypeMap.get(i).name(); + break; + case BINARY: + columnTypeName = ""; + break; + default: + columnTypeName = ""; + } + } + sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(");"); + String crateTableString = sb.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(crateTableString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(crateTableString); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + StringBuilder sb2 = new StringBuilder(); + sb2.append("INSERT INTO " + tableName + " "+ selectString); + String insertValueString = sb2.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(insertValueString); + } + stmt = null; + try { + stmt = this.con.createStatement(); + try { + Main.nrSuccessfulActions.addAndGet(1); + stmt.execute(insertValueString); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); + databaseColumns.add(column); + } + SQLite3Table table = new SQLite3Table(tableName, databaseColumns, null, false, false, false, false); + for (SQLite3Column c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private void dropTemporaryTable(String tableName) throws SQLException { + String dropString = "DROP TABLE " + tableName + ";"; + if (options.logEachSelect()) { + logger.writeCurrent(dropString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(dropString); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + } + + private boolean compareResult(Map> r1, Map> r2) { + if (r1.size() != r2.size()) { + return false; + } + for (Map.Entry < String, List > entry: r1.entrySet()) { + String currentKey = entry.getKey(); + if (!r2.containsKey(currentKey)) { + return false; + } + List v1= entry.getValue(); + List v2= r2.get(currentKey); + if (v1.size() != v2.size()) { + return false; + } + List v1Value = new ArrayList<>(v1.stream().map(c -> c.toString()).collect(Collectors.toList())); + List v2Value = new ArrayList<>(v2.stream().map(c -> c.toString()).collect(Collectors.toList())); + Collections.sort(v1Value); + Collections.sort(v2Value); + if (!v1Value.equals(v2Value)) { + return false; + } + } + return true; + } + + private Map getColumnTypeFromSelect(SQLite3Select select) { + List fetchColumns = select.getFetchColumns(); + List newFetchColumns = new ArrayList<>(); + for(SQLite3Expression column : fetchColumns) { + newFetchColumns.add(column); + SQLite3Alias columnAlias = (SQLite3Alias) column; + SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOrigonalExpression()); + newFetchColumns.add(typeofColumn); + } + SQLite3Select newSelect = new SQLite3Select(select); + newSelect.setFetchColumns(newFetchColumns); + Map> typeResult = null; + try { + typeResult = getQueryResult(SQLite3Visitor.asString(newSelect), state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + + if (typeResult == null) { + throw new IgnoreMeException(); + } + Map idxTypeMap = new HashMap<>(); + for (int i = 0; i * 2 < typeResult.size(); ++i) { + String columnName = "c" + String.valueOf(i * 2 + 1); + SQLite3Expression t = typeResult.get(columnName).get(0); + SQLite3TextConstant tString = (SQLite3TextConstant) t; + String typeName = tString.asString(); + SQLite3DataType cType = SQLite3DataType.getTypeFromName(typeName); + idxTypeMap.put(i, cType); + } + + return idxTypeMap; + } + + public boolean useSubquery() { + if (this.state.getDbmsSpecificOptions().coddTestModel.equals("random")) { + return Randomly.getBoolean(); + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("expression")) { + return false; + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("subquery")) { + return true; + } else { + System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); + System.exit(1); + return false; + } + } + + public boolean useCorrelatedSubquery() { + return Randomly.getBoolean(); + } + + public boolean testCommonTableExpression() { + return false; + } + public boolean testDerivedTable() { + return true; + } + public boolean testInsert() { + return false; + } + + @Override + public String getLastQueryString() { + return originalQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } +} diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index 87d91f451..8a343fed2 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -1,6 +1,23 @@ package sqlancer.sqlite3.schema; +import sqlancer.IgnoreMeException; + public enum SQLite3DataType { NULL, INT, TEXT, REAL, NONE, BINARY; + public static SQLite3DataType getTypeFromName(String name) { + if (name.equals("integer")) { + return INT; + } else if (name.equals("real")) { + return REAL; + } else if (name.equals("text")) { + return TEXT; + } else if (name.equals("blob")) { + return NONE; + } else if (name.equals("null")) { + return NULL; + } else { + throw new IgnoreMeException(); + } + } } From 2f0c1eef72f0d4de9661909362eeaa9a5b6057d2 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sun, 5 Jan 2025 21:09:59 +0800 Subject: [PATCH 649/856] fix a typo and remove some redundant code of CODDTest for sqlite3 --- .../sqlite3/SQLite3ExpectedValueVisitor.java | 2 +- .../sqlite3/SQLite3ToStringVisitor.java | 2 +- .../sqlite3/ast/SQLite3Expression.java | 10 +-- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 69 ++++--------------- 4 files changed, 22 insertions(+), 61 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 40f8ae20a..246105a5e 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -313,7 +313,7 @@ public void visit(SQLite3SetClause set) { @Override public void visit(SQLite3Alias alias) { print(alias); - print(alias.getOrigonalExpression()); + print(alias.getOriginalExpression()); print(alias.getAliasExpression()); } diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 44aa0d1e0..2ec206e11 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -512,7 +512,7 @@ public void visit(SQLite3SetClause set) { @Override public void visit(SQLite3Alias alias) { sb.append("("); - visit(alias.getOrigonalExpression()); + visit(alias.getOriginalExpression()); sb.append(")"); sb.append(" AS "); visit(alias.getAliasExpression()); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index a14432eec..6c4b2f9ff 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1593,11 +1593,11 @@ public SQLite3CollateSequence getExplicitCollateSequence() { public static class SQLite3Alias extends SQLite3Expression { - private SQLite3Expression origonalExpression; + private SQLite3Expression originalExpression; private SQLite3Expression aliasExpression; - public SQLite3Alias(SQLite3Expression origonalExpression, SQLite3Expression aliasExpression) { - this.origonalExpression = origonalExpression; + public SQLite3Alias(SQLite3Expression originalExpression, SQLite3Expression aliasExpression) { + this.originalExpression = originalExpression; this.aliasExpression = aliasExpression; } @@ -1606,8 +1606,8 @@ public SQLite3CollateSequence getExplicitCollateSequence() { return null; } - public SQLite3Expression getOrigonalExpression() { - return origonalExpression; + public SQLite3Expression getOriginalExpression() { + return originalExpression; } public SQLite3Expression getAliasExpression() { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index d20e6e9be..7118f4081 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -100,7 +100,7 @@ public void check() throws SQLException { SQLite3Select auxiliaryQuery = null; if (useSubqueryAsFoldedExpr) { if (useCorrelatedSubqueryAsFoldedExpr) { - auxiliaryQuery = genSelectWithCorrelatedSubquery(null, null); + auxiliaryQuery = genSelectWithCorrelatedSubquery(); auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); auxiliaryQueryResult.putAll(selectResult); @@ -110,7 +110,7 @@ public void check() throws SQLException { auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); } } else { - auxiliaryQuery = genSimpleSelect(null, null); + auxiliaryQuery = genSimpleSelect(); auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); auxiliaryQueryResult.putAll(selectResult); @@ -334,40 +334,18 @@ else if (Randomly.getBoolean()) { } // For expression test - private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression specificCondition) { + private SQLite3Select genSimpleSelect() { SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); - if (tempTable != null) { - randomTables.addTable(tempTable); - } - if (!useSubqueryAsFoldedExpr) { - for (SQLite3Table t : this.tablesFromOuterContext) { - randomTables.addTable(t); - } - if (this.joinsInExpr != null) { - for (Join j : this.joinsInExpr) { - SQLite3Table t = j.getTable(); - randomTables.removeTable(t); - } - } - } - List columns = randomTables.getColumns(); - if (!useSubqueryAsFoldedExpr && this.joinsInExpr != null) { - for (Join j : this.joinsInExpr) { - SQLite3Table t = j.getTable(); - columns.addAll(t.getColumns()); - } - } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); List tables = randomTables.getTables(); tablesFromOuterContext = randomTables.getTables(); - if (joinsInExpr == null) { - if (Randomly.getBooleanWithRatherLowProbability()) { - joinsInExpr = genJoinExpression(gen, tables, null, true); - } else { - joinsInExpr = new ArrayList(); - } + if (Randomly.getBooleanWithRatherLowProbability()) { + joinsInExpr = genJoinExpression(gen, tables, null, true); + } else { + joinsInExpr = new ArrayList(); } List tableRefs = SQLite3Common.getTableRefs(tables, s); @@ -378,10 +356,6 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression } SQLite3Expression whereCondition = gen.generateExpression(); - if (specificCondition != null) { - BinaryOperator operator = BinaryOperator.getRandomOperator(); - whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(whereCondition, specificCondition, operator); - } this.foldedExpr = whereCondition; List fetchColumns = new ArrayList<>(); @@ -401,11 +375,9 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression select.setFetchColumns(fetchColumns); - originalQueryString = SQLite3Visitor.asString(select); - Map> queryRes = null; try { - queryRes = getQueryResult(originalQueryString, state); + queryRes = getQueryResult(SQLite3Visitor.asString(select), state); } catch (SQLException e) { if (errors.errorIsExpected(e.getMessage())) { throw new IgnoreMeException(); @@ -429,7 +401,7 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression for (int i = 0; i < fetchColumns.size() - 1; ++i) { // do not put the last fetch column to values SQLite3Alias cAlias = (SQLite3Alias) fetchColumns.get(i); - SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOriginalExpression(); SQLite3Column column = cRef.getColumn(); String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); @@ -448,17 +420,10 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression return select; } - private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable, SQLite3Expression specificCondition) { - // do not support join now - this.joinsInExpr = new ArrayList(); - + private SQLite3Select genSelectWithCorrelatedSubquery() { SQLite3Tables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); SQLite3Tables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); - if (selectedTable != null) { - innerQueryRandomTables.addTable(selectedTable); - } - List innerQueryFromTables = new ArrayList<>(); for (SQLite3Table t : innerQueryRandomTables.getTables()) { if (!outerQueryRandomTables.isContained(t)) { @@ -494,10 +459,6 @@ private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable innerQuery.setFromList(innerQueryFromTables); SQLite3Expression innerQueryWhereCondition = gen.generateExpression(); - if (specificCondition != null) { - BinaryOperator operator = BinaryOperator.getRandomOperator(); - innerQueryWhereCondition = new SQLite3Expression.Sqlite3BinaryOperation(innerQueryWhereCondition, specificCondition, operator); - } innerQuery.setWhereClause(innerQueryWhereCondition); // use aggregate function in fetch column @@ -506,10 +467,10 @@ private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); if (Randomly.getBooleanWithRatherLowProbability()) { - List groupByClause = genGroupByClause(innerQueryColumns, specificCondition); + List groupByClause = genGroupByClause(innerQueryColumns, null); innerQuery.setGroupByClause(groupByClause); if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { - innerQuery.setHavingClause(genHavingClause(innerQueryColumns, specificCondition)); + innerQuery.setHavingClause(genHavingClause(innerQueryColumns, null)); } } @@ -566,7 +527,7 @@ private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable for (int i = 0; i < outerQueryFetchColumns.size() - 1; ++i) { // do not put the last fetch column to values SQLite3Alias cAlias = (SQLite3Alias) outerQueryFetchColumns.get(i); - SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOriginalExpression(); SQLite3Column column = cRef.getColumn(); String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); @@ -930,7 +891,7 @@ private Map getColumnTypeFromSelect(SQLite3Select sele for(SQLite3Expression column : fetchColumns) { newFetchColumns.add(column); SQLite3Alias columnAlias = (SQLite3Alias) column; - SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOrigonalExpression()); + SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOriginalExpression()); newFetchColumns.add(typeofColumn); } SQLite3Select newSelect = new SQLite3Select(select); From b8dbd9de91c5b8dbe2e2ca5b248b3bc59393ea84 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Tue, 14 Jan 2025 16:03:00 +0800 Subject: [PATCH 650/856] fix the false positive in CODDTest for SQLite --- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 7118f4081..c4e8d892a 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -94,6 +94,9 @@ public SQLite3CODDTestOracle(SQLite3GlobalState globalState) { public void check() throws SQLException { reproducer = null; + joinsInExpr = null; + tablesFromOuterContext.clear(); + useSubqueryAsFoldedExpr = useSubquery(); useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); @@ -168,8 +171,7 @@ else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQ foldedResult = getQueryResult(foldedQueryString, state); } // one column - else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability()) { - // else if (auxiliaryQueryResult.size() == 1 && false) { + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability() && testInOperator()) { // original query List columns = s.getRandomTableNonEmptyTables().getColumns(); SQLite3ColumnName selectedColumn = new SQLite3ColumnName(Randomly.fromList(columns), null); @@ -277,9 +279,11 @@ private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Express gen = new SQLite3ExpressionGenerator(state).setColumns(columns); List tables = randomTables.getTables(); List joinStatements = new ArrayList<>(); - if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { - joinStatements.addAll(this.joinsInExpr); - this.joinsInExpr = null; + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (this.joinsInExpr != null) { + joinStatements.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } } else if (Randomly.getBoolean()) { joinStatements = genJoinExpression(gen, tables, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); @@ -950,6 +954,9 @@ public boolean testDerivedTable() { public boolean testInsert() { return false; } + public boolean testInOperator() { + return false; + } @Override public String getLastQueryString() { From d9579b845b0469b8644b445c7b9fed27053f65eb Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sat, 22 Mar 2025 15:14:35 +0800 Subject: [PATCH 651/856] fix a typo --- .settings/org.eclipse.jdt.core.prefs | 9 ++++++--- .../sqlite3/SQLite3ExpectedValueVisitor.java | 10 +++++----- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 12 ++++++------ src/sqlancer/sqlite3/SQLite3Visitor.java | 8 ++++---- src/sqlancer/sqlite3/ast/SQLite3Expression.java | 4 ++-- src/sqlancer/sqlite3/ast/SQLite3Select.java | 12 ++++++------ .../sqlite3/oracle/SQLite3CODDTestOracle.java | 12 ++++++------ 7 files changed, 35 insertions(+), 32 deletions(-) diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 0a659c1a7..3af1c5ad8 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.compliance=11 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,6 +28,7 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -73,6 +74,7 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -110,5 +112,6 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=1.8 +org.eclipse.jdt.core.compiler.source=11 diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 246105a5e..594fe5c92 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -28,7 +28,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -318,10 +318,10 @@ public void visit(SQLite3Alias alias) { } @Override - public void visit(SQLite3WithClasure withClasure) { - print(withClasure); - print(withClasure.getLeft()); - print(withClasure.getRight()); + public void visit(SQLite3WithClause withClause) { + print(withClause); + print(withClause.getLeft()); + print(withClause.getRight()); } @Override diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 2ec206e11..202540eb6 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -36,7 +36,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; @@ -112,8 +112,8 @@ public void visit(SQLite3Select s, boolean inner) { if (inner) { sb.append("("); } - if (s.getWithClasure() != null) { - visit(s.getWithClasure()); + if (s.getWithClause() != null) { + visit(s.getWithClause()); sb.append(" "); } sb.append("SELECT "); @@ -519,11 +519,11 @@ public void visit(SQLite3Alias alias) { } @Override - public void visit(SQLite3WithClasure withClasure) { + public void visit(SQLite3WithClause withClause) { sb.append("WITH "); - visit(withClasure.getLeft()); + visit(withClause.getLeft()); sb.append(" AS "); - visit(withClasure.getRight()); + visit(withClause.getRight()); } @Override diff --git a/src/sqlancer/sqlite3/SQLite3Visitor.java b/src/sqlancer/sqlite3/SQLite3Visitor.java index d5ebce096..5e1e5dbf6 100644 --- a/src/sqlancer/sqlite3/SQLite3Visitor.java +++ b/src/sqlancer/sqlite3/SQLite3Visitor.java @@ -27,7 +27,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -140,7 +140,7 @@ default void visit(SQLite3PostfixUnaryOperation exp) { void visit(SQLite3Alias alias); - void visit(SQLite3WithClasure withClasure); + void visit(SQLite3WithClause withClause); void visit(SQLite3TableAndColumnRef tableAndColumnRef); @@ -217,8 +217,8 @@ default void visit(SQLite3Expression expr) { visit((SQLite3SetClause) expr); } else if (expr instanceof SQLite3Alias) { visit((SQLite3Alias) expr); - } else if (expr instanceof SQLite3WithClasure) { - visit((SQLite3WithClasure) expr); + } else if (expr instanceof SQLite3WithClause) { + visit((SQLite3WithClause) expr); } else if (expr instanceof SQLite3TableAndColumnRef) { visit((SQLite3TableAndColumnRef) expr); } else if (expr instanceof SQLite3Values) { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 6c4b2f9ff..12e5c71f1 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1563,12 +1563,12 @@ public boolean omitBracketsWhenPrinting() { } } - public static class SQLite3WithClasure extends SQLite3Expression { + public static class SQLite3WithClause extends SQLite3Expression { private SQLite3Expression left; private SQLite3Expression right; - public SQLite3WithClasure(SQLite3Expression left, SQLite3Expression right) { + public SQLite3WithClause(SQLite3Expression left, SQLite3Expression right) { this.left = left; this.right = right; } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 042aab856..6f1571df0 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -25,7 +25,7 @@ public class SQLite3Select extends SQLite3Expression private List fetchColumns = Collections.emptyList(); private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; - private SQLite3WithClasure withClause = null; + private SQLite3WithClause withClause = null; public SQLite3Select() { } @@ -165,15 +165,15 @@ public String asString() { return SQLite3Visitor.asString(this); } - public void setWithClasure(SQLite3WithClasure withClasure) { - this.withClause = withClasure; + public void setWithClause(SQLite3WithClause withClause) { + this.withClause = withClause; } - public void updateWithClasureRight(SQLite3Expression withClasureRight) { - this.withClause.updateRight(withClasureRight); + public void updateWithClauseRight(SQLite3Expression withClauseRight) { + this.withClause.updateRight(withClauseRight); } - public SQLite3Expression getWithClasure() { + public SQLite3Expression getWithClause() { return this.withClause; } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index c4e8d892a..17444026a 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -41,7 +41,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; @@ -202,8 +202,8 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); originalQuery = this.genSelectExpression(temporaryTable, null); SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); - SQLite3WithClasure withClasure = new SQLite3WithClasure(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); - originalQuery.setWithClasure(withClasure); + SQLite3WithClause withClause = new SQLite3WithClause(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); + originalQuery.setWithClause(withClause); originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); // folded query @@ -212,13 +212,13 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro // common table expression // folded query: WITH table AS VALUES () SQLite3Values values = new SQLite3Values(auxiliaryQueryResult, temporaryTable.getColumns()); - originalQuery.updateWithClasureRight(values); + originalQuery.updateWithClauseRight(values); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } else if (Randomly.getBoolean() && this.testDerivedTable()) { // derived table // folded query: SELECT FROM () AS table - originalQuery.setWithClasure(null); + originalQuery.setWithClause(null); SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); originalQuery.replaceFromTable(this.tempTableName, alias); @@ -229,7 +229,7 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro // folded query: CREATE the table and INSERT INTO table subquery try { this.createTemporaryTable(auxiliaryQuery, this.tempTableName); - originalQuery.setWithClasure(null); + originalQuery.setWithClause(null); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } finally { From 9a3a2e9bd8077451df53849d85217be65da44b3f Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sat, 22 Mar 2025 15:16:11 +0800 Subject: [PATCH 652/856] add comments for SQLite3ExpressionBag --- src/sqlancer/sqlite3/ast/SQLite3Expression.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 12e5c71f1..e5f1e2064 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1671,6 +1671,9 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } } + // The ExpressionBag is not a built-in SQL feature, + // but rather a utility class used in CODDTest's oracle construction + // to substitute expressions with their corresponding constant values. public static class SQLite3ExpressionBag extends SQLite3Expression { private SQLite3Expression innerExpr; From 9ead27ac349f81df77351fe78c371f1e253e0f30 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sat, 22 Mar 2025 15:19:40 +0800 Subject: [PATCH 653/856] remove deleteCharAt to simplify code logic --- .../sqlite3/SQLite3ToStringVisitor.java | 38 +++++++++---------- .../sqlite3/ast/SQLite3Expression.java | 11 ++++-- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 9 +++-- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 202540eb6..987d293e8 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -156,7 +156,7 @@ public void visit(SQLite3Select s, boolean inner) { visit(whereClause); sb.append(")"); } - if (!s.getGroupByClause().isEmpty()) { + if (s.getGroupByClause().size() > 0) { sb.append(" "); sb.append("GROUP BY "); visit(s.getGroupByClause()); @@ -539,7 +539,11 @@ public void visit(SQLite3Values values) { sb.append("(VALUES "); for (int i = 0; i < size; i++) { sb.append("("); + Boolean isFirstColumn = true; for (String name : columnNames) { + if (!isFirstColumn) { + sb.append(", "); + } if (vs.get(name).get(i).getDataType() == SQLite3DataType.NULL) { visit(vs.get(name).get(i)); } else { @@ -563,15 +567,13 @@ public void visit(SQLite3Values values) { throw new IgnoreMeException(); } } - + isFirstColumn = true; + } + sb.append(")"); + if (i < size - 1) { sb.append(", "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("), "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); } @@ -600,13 +602,11 @@ public void visit(SQLite3ResultMap tableSummary) { sb.append("("); for (int j = 0; j < columnRefs.size(); ++j) { visit(columnRefs.get(j)); - sb.append(" IS NULL AND "); + sb.append(" IS NULL"); + if (j < columnRefs.size() - 1) { + sb.append(" AND "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); return; } @@ -622,16 +622,14 @@ public void visit(SQLite3ResultMap tableSummary) { sb.append(" = "); sb.append(vs.get(columnNames.get(j)).get(i).toString()); } - sb.append(" AND "); + if (j < columnNames.size() - 1) { + sb.append(" AND "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("THEN "); + sb.append(" THEN "); visit(summary.get(i)); sb.append(" "); } sb.append("END "); } -} +} \ No newline at end of file diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index e5f1e2064..681566c03 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1631,12 +1631,14 @@ public String getString() { StringBuilder sb = new StringBuilder(); sb.append(table.getName()); sb.append("("); + Boolean isFirstColumn = true; for (SQLite3Column c : this.table.getColumns()) { + if (!isFirstColumn) { + sb.append(", "); + } sb.append(c.getName()); - sb.append(", "); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); return sb.toString(); } @@ -1671,6 +1673,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } } + // The ExpressionBag is not a built-in SQL feature, // but rather a utility class used in CODDTest's oracle construction // to substitute expressions with their corresponding constant values. @@ -1754,4 +1757,4 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } } -} +} \ No newline at end of file diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 17444026a..72d1b6072 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -781,10 +781,11 @@ private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName columnTypeName = ""; } } - sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + sb.append("c" + String.valueOf(i) + " " + columnTypeName); + if (i < columnNumber - 1) { + sb.append(", "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(");"); String crateTableString = sb.toString(); if (options.logEachSelect()) { @@ -967,4 +968,4 @@ public String getLastQueryString() { public Reproducer getLastReproducer() { return reproducer; } -} +} \ No newline at end of file From 967852ad31afcb697deb9808fe405960a49d540c Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Sun, 23 Mar 2025 07:03:25 +0000 Subject: [PATCH 654/856] Improve tests for schema classes --- test/sqlancer/TestCommonSchema.java | 34 +++++++++++++---------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/test/sqlancer/TestCommonSchema.java b/test/sqlancer/TestCommonSchema.java index 764e94ea4..80a57dd8c 100644 --- a/test/sqlancer/TestCommonSchema.java +++ b/test/sqlancer/TestCommonSchema.java @@ -42,8 +42,7 @@ static class TestIndex extends TableIndex { } private TestTable createTestTable(String name, List indexes, boolean isView, String... columns) { - List cols = Arrays.stream(columns) - .map(col -> new TestTableColumn(col, null, "VARCHAR")) + List cols = Arrays.stream(columns).map(col -> new TestTableColumn(col, null, "VARCHAR")) .collect(Collectors.toList()); return new TestTable(name, cols, indexes, isView); } @@ -59,11 +58,14 @@ private TestSchema createTestSchema(TestTable... tables) { @Test void testColumnManagement() { TestTable table = createTestTable("products", Collections.emptyList(), false, "sku", "price"); + List columnNames = table.getColumns().stream().map(TestTableColumn::getName) + .collect(Collectors.toList()); + List columnTypes = table.getColumns().stream().map(TestTableColumn::getType) + .collect(Collectors.toList()); TestTableColumn randomCol = table.getRandomColumn(); - assertEquals("sku", table.getColumns().get(0).getName()); - assertEquals("VARCHAR", table.getColumns().get(1).getType()); - + assertTrue(columnNames.containsAll(Set.of("sku", "price"))); + assertTrue(columnTypes.containsAll(Set.of("VARCHAR"))); assertTrue(table.getColumns().contains(randomCol)); } @@ -81,16 +83,14 @@ void testIndexManagement() { @Test void testViewManagement() { - TestTable view1 = createTestTable("v1", Collections.emptyList(),true, "col1"); - TestTable view2 = createTestTable("v2", Collections.emptyList(),true, "col2"); + TestTable view1 = createTestTable("v1", Collections.emptyList(), true, "col1"); + TestTable view2 = createTestTable("v2", Collections.emptyList(), true, "col2"); TestTable table = createTestTable("t1", Collections.emptyList(), false, "col3"); TestSchema schema = createTestSchema(view1, view2, table); - assertAll( - () -> assertEquals(2, schema.getViews().size(), "Should detect 2 views"), + assertAll(() -> assertEquals(2, schema.getViews().size(), "Should detect 2 views"), () -> assertEquals(1, schema.getDatabaseTablesWithoutViews().size(), "Should detect 1 normal table"), - () -> assertEquals("t1", schema.getDatabaseTablesWithoutViews().get(0).getName()) - ); + () -> assertEquals("t1", schema.getDatabaseTablesWithoutViews().get(0).getName())); } @Test @@ -117,12 +117,10 @@ void testObjectComparison() { TestTableColumn colA2 = new TestTableColumn("y", tableA, "INT"); TestTableColumn colB1 = new TestTableColumn("b", tableB, "TEXT"); - assertAll( - () -> assertTrue(colA1.compareTo(colA2) < 0, "Columns should be ordered by name"), + assertAll(() -> assertTrue(colA1.compareTo(colA2) < 0, "Columns should be ordered by name"), () -> assertTrue(colA1.compareTo(colB1) < 0, "Columns should be ordered by name"), () -> assertTrue(tableA.compareTo(tableB) > 0, "Tables should be ordered reverse-alphabetically"), - () -> assertEquals(0, tableA.compareTo(tableA), "Same table should be equal") - ); + () -> assertEquals(0, tableA.compareTo(tableA), "Same table should be equal")); } @Test @@ -135,12 +133,10 @@ void testEquality() { TestTableColumn col3 = new TestTableColumn("id", table2, "INT"); TestTableColumn col4 = new TestTableColumn("name", table1, "TEXT"); - assertAll( - () -> assertEquals(col1, col2, "Same table/column should be equal"), + assertAll(() -> assertEquals(col1, col2, "Same table/column should be equal"), () -> assertNotEquals(col1, col3, "Different tables should not be equal"), () -> assertNotEquals(col1, col4, "Different columns should not be equal"), - () -> assertNotEquals(col1, "invalid_object", "Different types should not be equal") - ); + () -> assertNotEquals(col1, "invalid_object", "Different types should not be equal")); } @Test From 9a4999434d805a3102917dcdf2a24a68d7fa21fc Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sat, 5 Apr 2025 17:08:17 +0800 Subject: [PATCH 655/856] [MySQL] Support generating CASE operators --- .../mysql/MySQLExpectedValueVisitor.java | 23 +++++++++++++ src/sqlancer/mysql/MySQLToStringVisitor.java | 27 +++++++++++++++ src/sqlancer/mysql/MySQLVisitor.java | 5 +++ src/sqlancer/mysql/ast/MySQLCaseOperator.java | 13 ++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 7 +++- .../mysql/MySQLExpectedValueVisitorTest.java | 33 +++++++++++++++++++ .../mysql/MySQLToStringVisitorTest.java | 27 +++++++++++++++ 7 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/mysql/ast/MySQLCaseOperator.java create mode 100644 test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index c9db08306..b0159fb27 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -1,11 +1,14 @@ package sqlancer.mysql; +import java.util.List; + import sqlancer.IgnoreMeException; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLCollate; import sqlancer.mysql.ast.MySQLColumnReference; @@ -172,4 +175,24 @@ public void visit(MySQLAggregate aggr) { // do nothing } + @Override + public void visit(MySQLCaseOperator caseOp) { + print(caseOp); + + if (caseOp.getSwitchCondition() != null) { + visit(caseOp.getSwitchCondition()); + } + + List whenConditions = caseOp.getConditions(); + List thenExpressions = caseOp.getExpressions(); + + for (int i = 0; i < whenConditions.size(); i++) { + visit(whenConditions.get(i)); + visit(thenExpressions.get(i)); + } + + if (caseOp.getElseExpr() != null) { + visit(caseOp.getElseExpr()); + } + } } diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index b459fe81e..be82d45b5 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -11,6 +11,7 @@ import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLCollate; import sqlancer.mysql.ast.MySQLColumnReference; @@ -345,4 +346,30 @@ public void visit(MySQLAggregate aggr) { } sb.append(")"); } + + @Override + public void visit(MySQLCaseOperator caseOp) { + sb.append("(CASE "); + + if (caseOp.getSwitchCondition() != null) { + visit(caseOp.getSwitchCondition()); + sb.append(" "); + } + + for (int i = 0; i < caseOp.getConditions().size(); i++) { + if (i > 0) { + sb.append(" "); + } + sb.append("WHEN "); + visit(caseOp.getConditions().get(i)); + sb.append(" THEN "); + visit(caseOp.getExpressions().get(i)); + } + + if (caseOp.getElseExpr() != null) { + sb.append(" ELSE "); + visit(caseOp.getElseExpr()); + } + sb.append(" END)"); + } } diff --git a/src/sqlancer/mysql/MySQLVisitor.java b/src/sqlancer/mysql/MySQLVisitor.java index e413f7668..12c93ecfc 100644 --- a/src/sqlancer/mysql/MySQLVisitor.java +++ b/src/sqlancer/mysql/MySQLVisitor.java @@ -5,6 +5,7 @@ import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLCollate; import sqlancer.mysql.ast.MySQLColumnReference; @@ -61,6 +62,8 @@ public interface MySQLVisitor { void visit(MySQLAggregate aggregate); + void visit(MySQLCaseOperator caseOp); + default void visit(MySQLExpression expr) { if (expr instanceof MySQLConstant) { visit((MySQLConstant) expr); @@ -100,6 +103,8 @@ default void visit(MySQLExpression expr) { visit((MySQLText) expr); } else if (expr instanceof MySQLAggregate) { visit((MySQLAggregate) expr); + } else if (expr instanceof MySQLCaseOperator) { + visit((MySQLCaseOperator) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/mysql/ast/MySQLCaseOperator.java b/src/sqlancer/mysql/ast/MySQLCaseOperator.java new file mode 100644 index 000000000..c648be5ec --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLCaseOperator.java @@ -0,0 +1,13 @@ +package sqlancer.mysql.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class MySQLCaseOperator extends NewCaseOperatorNode implements MySQLExpression { + + public MySQLCaseOperator(MySQLExpression switchCondition, List whenExprs, + List thenExprs, MySQLExpression elseExpr) { + super(switchCondition, whenExprs, thenExprs, elseExpr); + } +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index f23c48c29..1ce413db4 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -25,6 +25,7 @@ import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLBinaryOperation.MySQLBinaryOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLColumnReference; @@ -64,7 +65,7 @@ public MySQLExpressionGenerator setRowVal(MySQLRowValue rowVal) { private enum Actions { COLUMN, LITERAL, UNARY_PREFIX_OPERATION, UNARY_POSTFIX, COMPUTABLE_FUNCTION, BINARY_LOGICAL_OPERATOR, - BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, BINARY_OPERATION, EXISTS, BETWEEN_OPERATOR; + BINARY_COMPARISON_OPERATION, CAST, IN_OPERATION, BINARY_OPERATION, EXISTS, BETWEEN_OPERATOR, CASE_OPERATOR; } @Override @@ -117,6 +118,10 @@ public MySQLExpression generateExpression(int depth) { } return new MySQLBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), generateExpression(depth + 1)); + case CASE_OPERATOR: + int nr = Randomly.smallNumber() + 1; + return new MySQLCaseOperator(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); default: throw new AssertionError(); } diff --git a/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java b/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java new file mode 100644 index 000000000..05cd273b6 --- /dev/null +++ b/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java @@ -0,0 +1,33 @@ +package sqlancer.mysql; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; + +import org.junit.jupiter.api.Test; + +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; +import sqlancer.mysql.ast.MySQLExpression; + +public class MySQLExpectedValueVisitorTest { + + @Test + void testCaseOp() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, null); + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1)); + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11)); + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + MySQLExpectedValueVisitor visitor = new MySQLExpectedValueVisitor(); + + AssertionError err = assertThrows(AssertionError.class, + () -> visitor.visit(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr))); + + assertTrue(err.getMessage().contains("PQS not supported")); + } +} diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 5aa9b4e14..169aaa57b 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -7,7 +7,11 @@ import org.junit.jupiter.api.Test; import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; public class MySQLToStringVisitorTest { @@ -50,4 +54,27 @@ void visitAggregateWithDistinctToString() { MySQLAggregate.MySQLAggregateFunction.MAX_DISTINCT); assertEquals("MAX(DISTINCT a)", MySQLVisitor.asString(aggrMaxDistinct)); } + + @Test + void visitCaseWhenToString() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, null); + + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + assertEquals("(CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 0 END)", + MySQLVisitor.asString(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr))); + assertEquals("(CASE WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 0 END)", + MySQLVisitor.asString(new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr))); + assertEquals("(CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 END)", + MySQLVisitor.asString(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null))); + assertEquals("(CASE WHEN 1 THEN 11 WHEN 2 THEN 22 END)", + MySQLVisitor.asString(new MySQLCaseOperator(null, whenExprs, thenExprs, null))); + } } From 72311cbbb3be765829d98a7b766b11c74d8ae5f6 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sat, 5 Apr 2025 17:57:45 +0800 Subject: [PATCH 656/856] Format files --- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 1ce413db4..98641ab26 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -25,8 +25,8 @@ import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; import sqlancer.mysql.ast.MySQLBinaryOperation; -import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLBinaryOperation.MySQLBinaryOperator; +import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLComputableFunction; From 5fa0d837a09cb8c90ee6a021a40756090399a7f0 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sun, 6 Apr 2025 11:23:56 +0800 Subject: [PATCH 657/856] [MySQL] Add PQS for CASE expressions --- .../mysql/MySQLExpectedValueVisitor.java | 17 ++++++--- src/sqlancer/mysql/ast/MySQLCaseOperator.java | 35 +++++++++++++++++++ .../mysql/MySQLExpectedValueVisitorTest.java | 33 ----------------- 3 files changed, 47 insertions(+), 38 deletions(-) delete mode 100644 test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index b0159fb27..8ad2d8b2b 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -172,27 +172,34 @@ public void visit(MySQLText text) { @Override public void visit(MySQLAggregate aggr) { - // do nothing + // PQS is currently unsupported for aggregates. + throw new IgnoreMeException(); } @Override public void visit(MySQLCaseOperator caseOp) { print(caseOp); - if (caseOp.getSwitchCondition() != null) { - visit(caseOp.getSwitchCondition()); + MySQLExpression switchCondition = caseOp.getSwitchCondition(); + if (switchCondition != null) { + print(switchCondition); + visit(switchCondition); } List whenConditions = caseOp.getConditions(); List thenExpressions = caseOp.getExpressions(); for (int i = 0; i < whenConditions.size(); i++) { + print(whenConditions.get(i)); visit(whenConditions.get(i)); + print(thenExpressions.get(i)); visit(thenExpressions.get(i)); } - if (caseOp.getElseExpr() != null) { - visit(caseOp.getElseExpr()); + MySQLExpression elseExpr = caseOp.getElseExpr(); + if (elseExpr != null) { + print(elseExpr); + visit(elseExpr); } } } diff --git a/src/sqlancer/mysql/ast/MySQLCaseOperator.java b/src/sqlancer/mysql/ast/MySQLCaseOperator.java index c648be5ec..c91e33d7f 100644 --- a/src/sqlancer/mysql/ast/MySQLCaseOperator.java +++ b/src/sqlancer/mysql/ast/MySQLCaseOperator.java @@ -10,4 +10,39 @@ public MySQLCaseOperator(MySQLExpression switchCondition, List List thenExprs, MySQLExpression elseExpr) { super(switchCondition, whenExprs, thenExprs, elseExpr); } + + @Override + public MySQLConstant getExpectedValue() { + int nrConditions = getConditions().size(); + + MySQLExpression switchCondition = getSwitchCondition(); + List whenExprs = getConditions(); + List thenExprs = getExpressions(); + MySQLExpression elseExpr = getElseExpr(); + + if (switchCondition != null) { + MySQLConstant switchValue = switchCondition.getExpectedValue(); + + for (int i = 0; i < nrConditions; i++) { + MySQLConstant whenValue = whenExprs.get(i).getExpectedValue(); + MySQLConstant isConditionMatched = switchValue.isEquals(whenValue); + if (!isConditionMatched.isNull() && isConditionMatched.asBooleanNotNull()) { + return thenExprs.get(i).getExpectedValue(); + } + } + } else { + for (int i = 0; i < nrConditions; i++) { + MySQLConstant whenValue = whenExprs.get(i).getExpectedValue(); + if (!whenValue.isNull() && whenValue.asBooleanNotNull()) { + return thenExprs.get(i).getExpectedValue(); + } + } + } + + if (elseExpr != null) { + return elseExpr.getExpectedValue(); + } + + return null; + } } diff --git a/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java b/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java deleted file mode 100644 index 05cd273b6..000000000 --- a/test/sqlancer/mysql/MySQLExpectedValueVisitorTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package sqlancer.mysql; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.util.List; - -import org.junit.jupiter.api.Test; - -import sqlancer.mysql.ast.MySQLCaseOperator; -import sqlancer.mysql.ast.MySQLColumnReference; -import sqlancer.mysql.ast.MySQLConstant; -import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; -import sqlancer.mysql.ast.MySQLExpression; - -public class MySQLExpectedValueVisitorTest { - - @Test - void testCaseOp() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); - MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, null); - List whenExprs = List.of(MySQLIntConstant.createIntConstant(1)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); - - MySQLExpectedValueVisitor visitor = new MySQLExpectedValueVisitor(); - - AssertionError err = assertThrows(AssertionError.class, - () -> visitor.visit(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr))); - - assertTrue(err.getMessage().contains("PQS not supported")); - } -} From c6c8576a97e952f9421264287fdfca88d8a93743 Mon Sep 17 00:00:00 2001 From: Robins Tharakan Date: Tue, 8 Apr 2025 11:40:34 +0930 Subject: [PATCH 658/856] Add an expected error around ALTER TABLE ALTER COLUMN --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index ac2964661..f12927e90 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -206,6 +206,7 @@ public SQLQueryAdapter generate() { sb.append("DROP NOT NULL"); errors.add("is in a primary key"); errors.add("is an identity column"); + errors.add("is in index used as replica identity"); } break; case ALTER_COLUMN_SET_STATISTICS: From 089d2d00583c115ee5e3170d9f14bdb5cad27532 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 10 Apr 2025 16:58:50 +0800 Subject: [PATCH 659/856] Address the issues mentioned in the PR comments. --- .../common/schema/AbstractTables.java | 2 +- src/sqlancer/sqlite3/SQLite3Options.java | 14 +++++++- .../sqlite3/SQLite3ToStringVisitor.java | 34 +++---------------- src/sqlancer/sqlite3/ast/SQLite3Select.java | 2 ++ .../sqlite3/oracle/SQLite3CODDTestOracle.java | 6 ++-- .../sqlite3/schema/SQLite3DataType.java | 17 ++++++++++ 6 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/sqlancer/common/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java index c75cb30e1..67da6364e 100644 --- a/src/sqlancer/common/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -51,7 +51,7 @@ public void removeTable(T table) { } } - public Boolean isContained(T table) { + public boolean isContained(T table) { return this.tables.contains(table); } diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 47a80e832..6a5188337 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -79,8 +79,20 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") public int maxNumIndexes = 20; + public enum coddtest_model { + random, expression, subquery; + public boolean isRandom() { + return this == random; + } + public boolean isExpression() { + return this == expression; + } + public boolean isSubquery() { + return this == subquery; + } + } @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") - public String coddTestModel = "random"; + public coddtest_model coddTestModel = coddtest_model.random; @Override public List getTestOracleFactory() { diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 987d293e8..4b504a4be 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -5,7 +5,6 @@ import java.util.Map; import java.util.stream.Collectors; -import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; @@ -156,7 +155,7 @@ public void visit(SQLite3Select s, boolean inner) { visit(whereClause); sb.append(")"); } - if (s.getGroupByClause().size() > 0) { + if (!s.getGroupByClause().isEmpty()) { sb.append(" "); sb.append("GROUP BY "); visit(s.getGroupByClause()); @@ -550,22 +549,8 @@ public void visit(SQLite3Values values) { sb.append("(CAST("); visit(vs.get(name).get(i)); sb.append(" AS "); - switch(vs.get(name).get(i).getDataType()) { - case BINARY: - sb.append("BLOB))"); - break; - case INT: - sb.append("INT))"); - break; - case TEXT: - sb.append("TEXT))"); - break; - case REAL: - sb.append("REAL))"); - break; - default: - throw new IgnoreMeException(); - } + sb.append(vs.get(name).get(i).getDataType().toString()); + sb.append("))"); } isFirstColumn = true; } @@ -591,7 +576,7 @@ public void visit(SQLite3Typeof expr) { @Override public void visit(SQLite3ResultMap tableSummary) { - // we utlize CASE WHEN THEN END here + // We use the CASE WHEN THEN END expression to represent the result of an expression for each row in the table. SQLite3Values values = tableSummary.getValues(); List columnRefs = tableSummary.getColumns(); List summary = tableSummary.getSummary(); @@ -599,16 +584,7 @@ public void visit(SQLite3ResultMap tableSummary) { Map> vs = values.getValues(); int size = vs.get(vs.keySet().iterator().next()).size(); if (size == 0) { - sb.append("("); - for (int j = 0; j < columnRefs.size(); ++j) { - visit(columnRefs.get(j)); - sb.append(" IS NULL"); - if (j < columnRefs.size() - 1) { - sb.append(" AND "); - } - } - sb.append(")"); - return; + throw new AssertionError("The result of the expression must not be empty."); } List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); sb.append(" CASE "); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 6f1571df0..cf35b124a 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -177,6 +177,8 @@ public SQLite3Expression getWithClause() { return this.withClause; } + // This method is used in CODDTest to test subquery by replacing a table name + // in the SELECT clause with a derived table expression. public void replaceFromTable(String tableName, SQLite3Expression newFromExpression) { int replaceIdx = -1; for (int i = 0; i < fromList.size(); ++i) { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 72d1b6072..049ca7fe4 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -929,11 +929,11 @@ private Map getColumnTypeFromSelect(SQLite3Select sele } public boolean useSubquery() { - if (this.state.getDbmsSpecificOptions().coddTestModel.equals("random")) { + if (this.state.getDbmsSpecificOptions().coddTestModel.isRandom()) { return Randomly.getBoolean(); - } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("expression")) { + } else if (this.state.getDbmsSpecificOptions().coddTestModel.isExpression()) { return false; - } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("subquery")) { + } else if (this.state.getDbmsSpecificOptions().coddTestModel.isSubquery()) { return true; } else { System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index 8a343fed2..556976056 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -20,4 +20,21 @@ public static SQLite3DataType getTypeFromName(String name) { throw new IgnoreMeException(); } } + + public String toString() { + switch(this) { + case BINARY: + return "BLOB"; + case INT: + return "INT"; + case TEXT: + return "TEXT"; + case REAL: + return "REAL"; + case NULL: + return "NULL"; + default: + throw new IgnoreMeException(); + } + } } From 8e607cc46235142ee586adc0ad77b84feaae014e Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 10 Apr 2025 16:58:50 +0800 Subject: [PATCH 660/856] Address the issues mentioned in the PR comments. --- .settings/org.eclipse.jdt.core.prefs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 3af1c5ad8..0a659c1a7 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=11 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,7 +28,6 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore -org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -74,7 +73,6 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore -org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -112,6 +110,5 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=11 +org.eclipse.jdt.core.compiler.source=1.8 From a956eaaec4e2002cb4d21e01fbe8190d4058690d Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sun, 13 Apr 2025 10:48:56 +0800 Subject: [PATCH 661/856] [MySQL] Add tests for CaseOperator --- src/sqlancer/mysql/ast/MySQLCaseOperator.java | 2 +- .../mysql/MySQLToStringVisitorTest.java | 6 +- .../mysql/ast/MySQLCaseOperatorTest.java | 78 +++++++++++++++++++ 3 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java diff --git a/src/sqlancer/mysql/ast/MySQLCaseOperator.java b/src/sqlancer/mysql/ast/MySQLCaseOperator.java index c91e33d7f..ceb7fbbaa 100644 --- a/src/sqlancer/mysql/ast/MySQLCaseOperator.java +++ b/src/sqlancer/mysql/ast/MySQLCaseOperator.java @@ -43,6 +43,6 @@ public MySQLConstant getExpectedValue() { return elseExpr.getExpectedValue(); } - return null; + return MySQLConstant.createNullConstant(); } } diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 169aaa57b..7b30a2345 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -18,7 +18,7 @@ public class MySQLToStringVisitorTest { @Test void visitAggregateToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); - MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCount = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT); assertEquals("COUNT(a)", MySQLVisitor.asString(aggrCount)); @@ -36,7 +36,7 @@ void visitAggregateToString() { @Test void visitAggregateWithDistinctToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); - MySQLColumnReference aRef = new MySQLColumnReference(aCol, null); + MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCountDistinct = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT_DISTINCT); @@ -58,7 +58,7 @@ void visitAggregateWithDistinctToString() { @Test void visitCaseWhenToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); - MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, null); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); diff --git a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java new file mode 100644 index 000000000..6df384b50 --- /dev/null +++ b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java @@ -0,0 +1,78 @@ +package sqlancer.mysql.ast; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import java.util.List; + +import org.junit.jupiter.api.Test; + +import sqlancer.mysql.MySQLSchema; +import sqlancer.mysql.ast.MySQLConstant.MySQLIntConstant; + +public class MySQLCaseOperatorTest { + + @Test + void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createIntConstant(1)); + + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + MySQLCaseOperator caseOperator = new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr); + + assertEquals("11", caseOperator.getExpectedValue().toString()); + } + + @Test + void getExpectedValue_switchConditionHasNoMatches_ReturnsElse() { + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createNullConstant()); + + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + assertEquals("0", + new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr).getExpectedValue().toString()); + assertEquals("NULL", + new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null).getExpectedValue().toString()); + } + + @Test + void getExpectedValue_whenTrue_ReturnsThen() { + List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), + MySQLIntConstant.createIntConstant(2)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + MySQLCaseOperator caseOperator = new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr); + + assertEquals("11", caseOperator.getExpectedValue().toString()); + } + + @Test + void getExpectedValue_whenAllFalse_ReturnsElse() { + List whenExprs = List.of(MySQLIntConstant.createBoolean(false), + MySQLIntConstant.createBoolean(false)); + + List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), + MySQLIntConstant.createIntConstant(22)); + + MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); + + assertEquals("0", new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr).getExpectedValue().toString()); + assertEquals("NULL", new MySQLCaseOperator(null, whenExprs, thenExprs, null).getExpectedValue().toString()); + } +} From 49f36ed7fdf0c5f06406a7b646be9f3c73961e31 Mon Sep 17 00:00:00 2001 From: Emily Ong Date: Sun, 13 Apr 2025 11:39:18 +0800 Subject: [PATCH 662/856] Remove unneeded blank lines --- .../mysql/MySQLToStringVisitorTest.java | 3 --- .../mysql/ast/MySQLCaseOperatorTest.java | 27 ++++++------------- 2 files changed, 8 insertions(+), 22 deletions(-) diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 7b30a2345..3f991e3f1 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -59,13 +59,10 @@ void visitAggregateWithDistinctToString() { void visitCaseWhenToString() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); - List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); assertEquals("(CASE a WHEN 1 THEN 11 WHEN 2 THEN 22 ELSE 0 END)", diff --git a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java index 6df384b50..674ab027e 100644 --- a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java +++ b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java @@ -1,6 +1,8 @@ package sqlancer.mysql.ast; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + import java.util.List; import org.junit.jupiter.api.Test; @@ -14,65 +16,52 @@ public class MySQLCaseOperatorTest { void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createIntConstant(1)); - List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); MySQLCaseOperator caseOperator = new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr); - assertEquals("11", caseOperator.getExpectedValue().toString()); + assertEquals(11, caseOperator.getExpectedValue().getInt()); } @Test void getExpectedValue_switchConditionHasNoMatches_ReturnsElse() { MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createNullConstant()); - List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); - assertEquals("0", - new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr).getExpectedValue().toString()); - assertEquals("NULL", - new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null).getExpectedValue().toString()); + assertEquals(0, new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, elseExpr).getExpectedValue().getInt()); + assertTrue(new MySQLCaseOperator(switchExpr, whenExprs, thenExprs, null).getExpectedValue().isNull()); } @Test void getExpectedValue_whenTrue_ReturnsThen() { List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); - MySQLCaseOperator caseOperator = new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr); - assertEquals("11", caseOperator.getExpectedValue().toString()); + assertEquals(11, caseOperator.getExpectedValue().getInt()); } @Test void getExpectedValue_whenAllFalse_ReturnsElse() { List whenExprs = List.of(MySQLIntConstant.createBoolean(false), MySQLIntConstant.createBoolean(false)); - List thenExprs = List.of(MySQLIntConstant.createIntConstant(11), MySQLIntConstant.createIntConstant(22)); - MySQLConstant elseExpr = MySQLConstant.createIntConstant(0); - assertEquals("0", new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr).getExpectedValue().toString()); - assertEquals("NULL", new MySQLCaseOperator(null, whenExprs, thenExprs, null).getExpectedValue().toString()); + assertEquals(0, new MySQLCaseOperator(null, whenExprs, thenExprs, elseExpr).getExpectedValue().getInt()); + assertTrue(new MySQLCaseOperator(null, whenExprs, thenExprs, null).getExpectedValue().isNull()); } } From 4faa1c9741cf186e5b09d032e6491d3d8bcc4c19 Mon Sep 17 00:00:00 2001 From: liyxiris Date: Sun, 14 Jan 2024 20:21:16 +0800 Subject: [PATCH 663/856] [Hive] Support TLP oracle --- .github/workflows/main.yml | 36 ++ .gitignore | 4 +- pom.xml | 10 + src/sqlancer/Main.java | 2 + src/sqlancer/hive/HiveErrors.java | 40 +++ src/sqlancer/hive/HiveGlobalState.java | 11 + src/sqlancer/hive/HiveOptions.java | 35 ++ src/sqlancer/hive/HiveProvider.java | 121 +++++++ src/sqlancer/hive/HiveSchema.java | 114 ++++++ src/sqlancer/hive/HiveToStringVisitor.java | 115 ++++++ .../hive/ast/HiveBetweenOperation.java | 12 + .../hive/ast/HiveBinaryOperation.java | 12 + src/sqlancer/hive/ast/HiveCaseOperation.java | 13 + src/sqlancer/hive/ast/HiveCastOperation.java | 25 ++ .../hive/ast/HiveColumnReference.java | 11 + src/sqlancer/hive/ast/HiveConstant.java | 215 +++++++++++ src/sqlancer/hive/ast/HiveExpression.java | 7 + src/sqlancer/hive/ast/HiveFunction.java | 13 + src/sqlancer/hive/ast/HiveInOperation.java | 12 + src/sqlancer/hive/ast/HiveJoin.java | 48 +++ src/sqlancer/hive/ast/HiveOrderingTerm.java | 10 + src/sqlancer/hive/ast/HiveSelect.java | 41 +++ src/sqlancer/hive/ast/HiveTableReference.java | 13 + .../hive/ast/HiveUnaryPostfixOperation.java | 13 + .../hive/ast/HiveUnaryPrefixOperation.java | 13 + .../hive/gen/HiveExpressionGenerator.java | 334 ++++++++++++++++++ .../hive/gen/HiveInsertGenerator.java | 52 +++ src/sqlancer/hive/gen/HiveTableGenerator.java | 125 +++++++ src/sqlancer/hive/oracle/HiveTLPBase.java | 68 ++++ .../hive/oracle/HiveTLPWhereOracle.java | 42 +++ test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestHiveTLP.java | 20 ++ 32 files changed, 1585 insertions(+), 3 deletions(-) create mode 100644 src/sqlancer/hive/HiveErrors.java create mode 100644 src/sqlancer/hive/HiveGlobalState.java create mode 100644 src/sqlancer/hive/HiveOptions.java create mode 100644 src/sqlancer/hive/HiveProvider.java create mode 100644 src/sqlancer/hive/HiveSchema.java create mode 100644 src/sqlancer/hive/HiveToStringVisitor.java create mode 100644 src/sqlancer/hive/ast/HiveBetweenOperation.java create mode 100644 src/sqlancer/hive/ast/HiveBinaryOperation.java create mode 100644 src/sqlancer/hive/ast/HiveCaseOperation.java create mode 100644 src/sqlancer/hive/ast/HiveCastOperation.java create mode 100644 src/sqlancer/hive/ast/HiveColumnReference.java create mode 100644 src/sqlancer/hive/ast/HiveConstant.java create mode 100644 src/sqlancer/hive/ast/HiveExpression.java create mode 100644 src/sqlancer/hive/ast/HiveFunction.java create mode 100644 src/sqlancer/hive/ast/HiveInOperation.java create mode 100644 src/sqlancer/hive/ast/HiveJoin.java create mode 100644 src/sqlancer/hive/ast/HiveOrderingTerm.java create mode 100644 src/sqlancer/hive/ast/HiveSelect.java create mode 100644 src/sqlancer/hive/ast/HiveTableReference.java create mode 100644 src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java create mode 100644 src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java create mode 100644 src/sqlancer/hive/gen/HiveExpressionGenerator.java create mode 100644 src/sqlancer/hive/gen/HiveInsertGenerator.java create mode 100644 src/sqlancer/hive/gen/HiveTableGenerator.java create mode 100644 src/sqlancer/hive/oracle/HiveTLPBase.java create mode 100644 src/sqlancer/hive/oracle/HiveTLPWhereOracle.java create mode 100644 test/sqlancer/dbms/TestHiveTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1744f6a3c..8ea1059a6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -297,6 +297,42 @@ jobs: - name: Run Tests run: mvn -Dtest=TestH2 test + hive: + name: DBMS Tests (Hive) + runs-on: ubuntu-latest + services: + metastore: + image: apache/hive:4.0.1 + env: + SERVICE_NAME: 'metastore' + ports: + - 9083:9083 + volumes: + - warehouse:/opt/hive/data/warehouse + hiveserver2: + image: apache/hive:4.0.1 + env: + SERVICE_NAME: 'hiveserver2' + ports: + - 10000:10000 + - 10002:10002 + volumes: + - warehouse:/opt/hive/data/warehouse + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + - name: Build SQLancer + run: mvn -B package -DskipTests=true + - name: Run Tests + run: HIVE_AVAILABLE=true mvn -Dtest=TestHiveTLP test + hsqldb: name: DBMS Tests (HSQLB) runs-on: ubuntu-latest diff --git a/.gitignore b/.gitignore index ff30cb6de..d7cbeb55f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,6 @@ target/ .classpath -.settings/org.eclipse.core.resources.prefs -.settings/org.eclipse.m2e.core.prefs -.settings/org.eclipse.jdt.core.prefs +.settings/ .vscode .project .checkstyle diff --git a/pom.xml b/pom.xml index 836d84fa5..1894cae67 100644 --- a/pom.xml +++ b/pom.xml @@ -377,6 +377,16 @@ org.apache.arrow flight-sql-jdbc-driver 16.1.0 + + + org.apache.hive + hive-jdbc + 3.1.3 + + + org.apache.hive + hive-serde + 3.1.3 diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 7df750bf1..eb3ad7f0c 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -34,6 +34,7 @@ import sqlancer.doris.DorisProvider; import sqlancer.duckdb.DuckDBProvider; import sqlancer.h2.H2Provider; +import sqlancer.hive.HiveProvider; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.mariadb.MariaDBProvider; import sqlancer.materialize.MaterializeProvider; @@ -735,6 +736,7 @@ private static void checkForIssue799(List> providers) providers.add(new DorisProvider()); providers.add(new DuckDBProvider()); providers.add(new H2Provider()); + providers.add(new HiveProvider()); providers.add(new HSQLDBProvider()); providers.add(new MariaDBProvider()); providers.add(new MaterializeProvider()); diff --git a/src/sqlancer/hive/HiveErrors.java b/src/sqlancer/hive/HiveErrors.java new file mode 100644 index 000000000..b061c1071 --- /dev/null +++ b/src/sqlancer/hive/HiveErrors.java @@ -0,0 +1,40 @@ +package sqlancer.hive; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public class HiveErrors { + + private HiveErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("cannot recognize input near"); + errors.add("Argument type mismatch"); + errors.add("Error while compiling statement"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("Either CHECK or NOT NULL constraint violated!"); + errors.add("Error running query"); + errors.add("is different from preceding arguments"); + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } +} diff --git a/src/sqlancer/hive/HiveGlobalState.java b/src/sqlancer/hive/HiveGlobalState.java new file mode 100644 index 000000000..a362c1f88 --- /dev/null +++ b/src/sqlancer/hive/HiveGlobalState.java @@ -0,0 +1,11 @@ +package sqlancer.hive; + +import sqlancer.SQLGlobalState; + +public class HiveGlobalState extends SQLGlobalState { + + @Override + protected HiveSchema readSchema() throws Exception { + return HiveSchema.fromConnection(getConnection(), getDatabaseName()); + } +} diff --git a/src/sqlancer/hive/HiveOptions.java b/src/sqlancer/hive/HiveOptions.java new file mode 100644 index 000000000..fff2d74b1 --- /dev/null +++ b/src/sqlancer/hive/HiveOptions.java @@ -0,0 +1,35 @@ +package sqlancer.hive; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; +import sqlancer.common.oracle.TestOracle; +import sqlancer.DBMSSpecificOptions; +import sqlancer.hive.oracle.HiveTLPWhereOracle; +import sqlancer.OracleFactory; + +import java.util.Arrays; +import java.util.List; + +@Parameters(separators = "=", commandDescription = "Hive (default port: " + HiveOptions.DEFAULT_PORT + + ", default host: " + HiveOptions.DEFAULT_HOST + ")") +public class HiveOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 10000; + + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(HiveOracleFactory.TLPWhere); + + public enum HiveOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(HiveGlobalState globalState) { + return new HiveTLPWhereOracle(globalState); + } + }; + } + + @Override + public List getTestOracleFactory() { + return oracle; + } +} diff --git a/src/sqlancer/hive/HiveProvider.java b/src/sqlancer/hive/HiveProvider.java new file mode 100644 index 000000000..cf1df5410 --- /dev/null +++ b/src/sqlancer/hive/HiveProvider.java @@ -0,0 +1,121 @@ +package sqlancer.hive; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.hive.gen.HiveInsertGenerator; +import sqlancer.hive.gen.HiveTableGenerator; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +@AutoService(DatabaseProvider.class) +public class HiveProvider extends SQLProviderAdapter { + + public HiveProvider() { + super(HiveGlobalState.class, HiveOptions.class); + } + + public enum Action implements AbstractAction { + + INSERT(HiveInsertGenerator::getQuery); + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(HiveGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(HiveGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + default: + throw new AssertionError(a); + } + } + + @Override + public void generateDatabase(HiveGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + String tableName = globalState.getSchema().getFreeTableName(); + SQLQueryAdapter qt = HiveTableGenerator.generate(globalState, tableName); + success = globalState.executeStatement(qt); + } while(!success); + } + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); // TODO + } + + StatementExecutor se = new StatementExecutor( + globalState, Action.values(), + HiveProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(HiveGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + if (host == null) { + host = HiveOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = HiveOptions.DEFAULT_PORT; + } + + String databaseName = globalState.getDatabaseName(); + + String url = String.format("jdbc:hive2://%s:%d/%s", host, port, "default"); + Connection con = DriverManager.getConnection(url, username, password); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + globalState.getState().logStatement("CREATE DATABASE " + databaseName); + globalState.getState().logStatement("USE " + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + try (Statement s = con.createStatement()) { + s.execute("USE " + databaseName); + } + con.close(); + con = DriverManager.getConnection( + String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName, + username, password)); + + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "hive"; + } +} diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java new file mode 100644 index 000000000..018293093 --- /dev/null +++ b/src/sqlancer/hive/HiveSchema.java @@ -0,0 +1,114 @@ +package sqlancer.hive; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.DBMSCommon; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.hive.HiveSchema.HiveTable; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class HiveSchema extends AbstractSchema { + + public enum HiveDataType { + + // TODO: support more types, e.g. TIMESTAMP, DATE, VARCHAR, CHAR, BINARY, ARRAY, MAP, STRUCT, UNIONTYPE... + INT, FLOAT, DECIMAL, STRING, BOOLEAN; + + public static HiveDataType getRandomType() { + return Randomly.fromList(Arrays.asList(values())); + } + } + + public static class HiveColumn extends AbstractTableColumn { + + public HiveColumn(String name, HiveTable table, HiveDataType type) { + super(name, table, type); + } + } + + public static class HiveTables extends AbstractTables { + + public HiveTables(List tables) { + super(tables); + } + } + + public static class HiveTable extends AbstractRelationalTable { + + public HiveTable(String name, List columns, boolean isView) { + super(name, columns, Collections.emptyList(), isView); + } + } + + public HiveSchema(List databaseTables) { + super(databaseTables); + } + + public static HiveSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + // TODO: discover built-in tables + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + if (DBMSCommon.matchesIndexName(tableName)) { + continue; // TODO: unexpected? + } + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.startsWith("v"); + HiveTable t = new HiveTable(tableName, databaseColumns, isView); + for (HiveColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new HiveSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet tableRs = s.executeQuery("SHOW TABLES"); + while (tableRs.next()) { + String tableName = tableRs.getString(1); + tableNames.add(tableName); + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("DESCRIBE %s", tableName))) { + while (rs.next()) { + String columnName = rs.getString("col_name"); + String dataType = rs.getString("data_type"); + HiveColumn c = new HiveColumn(columnName, null, getColumnType(dataType.toUpperCase())); + columns.add(c); + } + } + } + return columns; + } + + private static HiveDataType getColumnType(String typeString) { + if (typeString.startsWith("DECIMAL")) { + return HiveDataType.DECIMAL; + } + return HiveDataType.valueOf(typeString.toUpperCase()); + } + + public HiveTables getRandomTableNonEmptyTables() { + return new HiveTables(Randomly.nonEmptySubset(getDatabaseTables())); + } +} diff --git a/src/sqlancer/hive/HiveToStringVisitor.java b/src/sqlancer/hive/HiveToStringVisitor.java new file mode 100644 index 000000000..3815561d8 --- /dev/null +++ b/src/sqlancer/hive/HiveToStringVisitor.java @@ -0,0 +1,115 @@ +package sqlancer.hive; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hive.ast.HiveConstant; +import sqlancer.hive.ast.HiveExpression; +import sqlancer.hive.ast.HiveJoin; +import sqlancer.hive.ast.HiveSelect; +import sqlancer.hive.ast.HiveCastOperation; + +public class HiveToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(HiveExpression expr) { + if (expr instanceof HiveConstant) { + visit((HiveConstant) expr); + } else if (expr instanceof HiveSelect) { + visit((HiveSelect) expr); + } else if (expr instanceof HiveJoin) { + visit((HiveJoin) expr); + } else if (expr instanceof HiveCastOperation) { + visit((HiveCastOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(HiveConstant constant) { + sb.append(constant.toString()); + } + + private void visit(HiveSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + private void visit(HiveJoin join) { + switch (join.getJoinType()) { + case INNER: + sb.append(" INNER JOIN "); + break; + case LEFT_OUTER: + sb.append(" LEFT JOIN "); + break; + case RIGHT_OUTER: + sb.append(" RIGHT JOIN "); + break; + case FULL_OUTER: + sb.append(" FULL JOIN "); + break; + case LEFT_SEMI: + sb.append(" LEFT SEMI JOIN "); + break; + case CROSS: + sb.append(" CROSS JOIN "); + break; + default: + throw new UnsupportedOperationException(); + } + visit((TableReferenceNode) join.getRightTable()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + private void visit(HiveCastOperation cast) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + sb.append(cast.getType()); + sb.append(")"); + } + + public static String asString(HiveExpression expr) { + HiveToStringVisitor visitor = new HiveToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} diff --git a/src/sqlancer/hive/ast/HiveBetweenOperation.java b/src/sqlancer/hive/ast/HiveBetweenOperation.java new file mode 100644 index 000000000..a65ef70d2 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveBetweenOperation.java @@ -0,0 +1,12 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class HiveBetweenOperation extends NewBetweenOperatorNode + implements HiveExpression { + + public HiveBetweenOperation(HiveExpression left, HiveExpression middle, HiveExpression right, + boolean isTrue) { + super(left, middle, right, isTrue); + } +} diff --git a/src/sqlancer/hive/ast/HiveBinaryOperation.java b/src/sqlancer/hive/ast/HiveBinaryOperation.java new file mode 100644 index 000000000..61d9cf104 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveBinaryOperation.java @@ -0,0 +1,12 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class HiveBinaryOperation extends NewBinaryOperatorNode + implements HiveExpression { + + public HiveBinaryOperation(HiveExpression left, HiveExpression right, Operator op) { + super(left, right, op); + } +} diff --git a/src/sqlancer/hive/ast/HiveCaseOperation.java b/src/sqlancer/hive/ast/HiveCaseOperation.java new file mode 100644 index 000000000..2a52a5c45 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveCaseOperation.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class HiveCaseOperation extends NewCaseOperatorNode implements HiveExpression { + + public HiveCaseOperation(HiveExpression switchCondition, List conditions, + List expressions, HiveExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/hive/ast/HiveCastOperation.java b/src/sqlancer/hive/ast/HiveCastOperation.java new file mode 100644 index 000000000..ae2fbe435 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveCastOperation.java @@ -0,0 +1,25 @@ +package sqlancer.hive.ast; + +import sqlancer.hive.HiveSchema.HiveDataType; + +public class HiveCastOperation implements HiveExpression { + + private final HiveExpression expression; + private final HiveDataType type; + + public HiveCastOperation(HiveExpression expression, HiveDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + public HiveExpression getExpression() { + return expression; + } + + public HiveDataType getType() { + return type; + } +} diff --git a/src/sqlancer/hive/ast/HiveColumnReference.java b/src/sqlancer/hive/ast/HiveColumnReference.java new file mode 100644 index 000000000..c3237955f --- /dev/null +++ b/src/sqlancer/hive/ast/HiveColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.hive.HiveSchema.HiveColumn; + +public class HiveColumnReference extends ColumnReferenceNode implements HiveExpression { + + public HiveColumnReference(HiveColumn column) { + super(column); + } +} diff --git a/src/sqlancer/hive/ast/HiveConstant.java b/src/sqlancer/hive/ast/HiveConstant.java new file mode 100644 index 000000000..ed3ed591d --- /dev/null +++ b/src/sqlancer/hive/ast/HiveConstant.java @@ -0,0 +1,215 @@ +package sqlancer.hive.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public abstract class HiveConstant implements HiveExpression { + + public boolean isNull() { + return false; + } + + public static class HiveNullConstant extends HiveConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public String toString() { + return "NULL"; + } + } + + public static class HiveIntConstant extends HiveConstant { + + private final long value; + + public HiveIntConstant(long value) { + this.value = value; + } + + public long getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + // public static class HiveFloatConstant extends HiveConstant { + + // private final float value; + + // public HiveFloatConstant(float value) { + // this.value = value; + // } + + // public float getValue() { + // return value; + // } + + // @Override + // public String toString() { + // if (value == Float.POSITIVE_INFINITY) { + // return "'+Inf'"; + // } else if (value == Float.NEGATIVE_INFINITY) { + // return "'-Inf'"; + // } + // return String.valueOf(value); + // } + // } + + public static class HiveDoubleConstant extends HiveConstant { + + private final double value; + + public HiveDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "'+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "'-Inf'"; + } + return String.valueOf(value); + } + } + + public static class HiveDecimalConstant extends HiveConstant { + + private final BigDecimal value; + + public HiveDecimalConstant(BigDecimal value) { + this.value = value; + } + + public BigDecimal getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class HiveTimestampConstant extends HiveConstant { + + private final String textRepr; + + public HiveTimestampConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + } + + public static class HiveDateConstant extends HiveConstant { + + private final String textRepr; + + public HiveDateConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + } + + public static class StringConstant extends HiveConstant { + + private final String value; + + public StringConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''") + "'"; + } + } + + public static class HiveBooleanConstant extends HiveConstant { + + private final boolean value; + + public HiveBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static HiveConstant createNullConstant() { + return new HiveNullConstant(); + } + + public static HiveConstant createIntConstant(long value) { + return new HiveIntConstant(value); + } + + public static HiveConstant createDoubleConstant(double value) { + return new HiveDoubleConstant(value); + } + + public static HiveConstant createDecimalConstant(BigDecimal value) { + return new HiveDecimalConstant(value); + } + + public static HiveConstant createTimestampConstant(long value) { + return new HiveTimestampConstant(value); + } + + public static HiveConstant createDateConstant(long value) { + return new HiveDateConstant(value); + } + + public static HiveConstant createStringConstant(String value) { + return new StringConstant(value); + } + + public static HiveConstant createBooleanConstant(boolean value) { + return new HiveBooleanConstant(value); + } +} diff --git a/src/sqlancer/hive/ast/HiveExpression.java b/src/sqlancer/hive/ast/HiveExpression.java new file mode 100644 index 000000000..40842e181 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveExpression.java @@ -0,0 +1,7 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.hive.HiveSchema.HiveColumn; + +public interface HiveExpression extends Expression { +} diff --git a/src/sqlancer/hive/ast/HiveFunction.java b/src/sqlancer/hive/ast/HiveFunction.java new file mode 100644 index 000000000..507015259 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveFunction.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class HiveFunction extends NewFunctionNode implements HiveExpression { + + public HiveFunction(List args, F func) { + super(args, func); + } + +} diff --git a/src/sqlancer/hive/ast/HiveInOperation.java b/src/sqlancer/hive/ast/HiveInOperation.java new file mode 100644 index 000000000..ccffde58c --- /dev/null +++ b/src/sqlancer/hive/ast/HiveInOperation.java @@ -0,0 +1,12 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class HiveInOperation extends NewInOperatorNode implements HiveExpression { + + public HiveInOperation(HiveExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} diff --git a/src/sqlancer/hive/ast/HiveJoin.java b/src/sqlancer/hive/ast/HiveJoin.java new file mode 100644 index 000000000..fc6b7a200 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveJoin.java @@ -0,0 +1,48 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.Join; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; + +public class HiveJoin implements HiveExpression, Join { + + private final HiveTableReference leftTable; + private final HiveTableReference rightTable; + private final JoinType joinType; + private HiveExpression onClause; + + // TODO: test map-join optimization + + public enum JoinType { + INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, CROSS; + } + + public HiveJoin(HiveTableReference leftTable, HiveTableReference rightTable, JoinType joinType, + HiveExpression onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onClause = onClause; + } + + public HiveTableReference getLeftTable() { + return leftTable; + } + + public HiveTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public HiveExpression getOnClause() { + return onClause; + } + + @Override + public void setOnClause(HiveExpression onClause) { + this.onClause = onClause; + } +} diff --git a/src/sqlancer/hive/ast/HiveOrderingTerm.java b/src/sqlancer/hive/ast/HiveOrderingTerm.java new file mode 100644 index 000000000..70fef52ad --- /dev/null +++ b/src/sqlancer/hive/ast/HiveOrderingTerm.java @@ -0,0 +1,10 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class HiveOrderingTerm extends NewOrderingTerm implements HiveExpression { + + public HiveOrderingTerm(HiveExpression expr, Ordering ordering) { + super(expr, ordering); + } +} diff --git a/src/sqlancer/hive/ast/HiveSelect.java b/src/sqlancer/hive/ast/HiveSelect.java new file mode 100644 index 000000000..ffd5e8fbd --- /dev/null +++ b/src/sqlancer/hive/ast/HiveSelect.java @@ -0,0 +1,41 @@ +package sqlancer.hive.ast; + +import java.util.List; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; + +public class HiveSelect extends SelectBase + implements Select, HiveExpression { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'setJoinClauses'"); + } + + @Override + public List getJoinClauses() { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'getJoinClauses'"); + } + + @Override + public String asString() { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'asString'"); + } + +} diff --git a/src/sqlancer/hive/ast/HiveTableReference.java b/src/sqlancer/hive/ast/HiveTableReference.java new file mode 100644 index 000000000..a0955f86c --- /dev/null +++ b/src/sqlancer/hive/ast/HiveTableReference.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hive.HiveSchema; + +public class HiveTableReference extends TableReferenceNode + implements HiveExpression { + + public HiveTableReference(HiveSchema.HiveTable table) { + super(table); + } + +} \ No newline at end of file diff --git a/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java new file mode 100644 index 000000000..c32a40847 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class HiveUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements HiveExpression { + + public HiveUnaryPostfixOperation(HiveExpression expr, Operator op) { + super(expr, op); + } + +} diff --git a/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java new file mode 100644 index 000000000..9312c4c26 --- /dev/null +++ b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java @@ -0,0 +1,13 @@ +package sqlancer.hive.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class HiveUnaryPrefixOperation extends NewUnaryPrefixOperatorNode + implements HiveExpression { + + public HiveUnaryPrefixOperation(HiveExpression expr, Operator op) { + super(expr, op); + } + +} diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java new file mode 100644 index 000000000..d8949a72c --- /dev/null +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -0,0 +1,334 @@ +package sqlancer.hive.gen; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema.*; +import sqlancer.hive.ast.HiveBetweenOperation; +import sqlancer.hive.ast.HiveBinaryOperation; +import sqlancer.hive.ast.HiveCaseOperation; +import sqlancer.hive.ast.HiveCastOperation; +import sqlancer.hive.ast.HiveColumnReference; +import sqlancer.hive.ast.HiveConstant; +import sqlancer.hive.ast.HiveExpression; +import sqlancer.hive.ast.HiveFunction; +import sqlancer.hive.ast.HiveInOperation; +import sqlancer.hive.ast.HiveOrderingTerm; +import sqlancer.hive.ast.HiveUnaryPrefixOperation; +import sqlancer.hive.ast.HiveUnaryPostfixOperation; +import sqlancer.common.ast.BinaryOperatorNode.Operator; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class HiveExpressionGenerator extends UntypedExpressionGenerator { + + private final HiveGlobalState globalState; + + private enum Expression { + // TODO: add or delete expressions. + UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, + CAST, FUNC, BETWEEN, IN, CASE; + } + + private enum ConstantType { + // TODO: add DECIMAL, DATE, TIMESTAMP, BINARY,... + STRING, INT, DOUBLE, BOOLEAN, NULL + } + + public HiveExpressionGenerator(HiveGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public HiveExpression negatePredicate(HiveExpression predicate) { + return new HiveUnaryPrefixOperation(predicate, HiveUnaryPrefixOperator.NOT); + } + + @Override + public HiveExpression isNull(HiveExpression expr) { + return new HiveUnaryPostfixOperation(expr, HiveUnaryPostfixOperator.IS_NULL); + } + + @Override + protected HiveExpression generateExpression(int depth) { + // TODO: randomly cast some types like what PostgresExpressionGenerator does? + return generateExpressionInternal(depth); + } + + private HiveExpression generateExpressionInternal(int depth) throws AssertionError { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBooleanWithRatherLowProbability()) { + allowAggregates= false; // aggregate function calls cannot be nested + HiveAggregateFunction aggregate = HiveAggregateFunction.getRandom(); + return new HiveFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + // TODO: remove some of the possible expression types according to options. + + Expression expr = Randomly.fromList(possibleOptions); + switch (expr) { + case UNARY_PREFIX: + return new HiveUnaryPrefixOperation(generateExpression(depth + 1), + HiveUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new HiveUnaryPostfixOperation(generateExpression(depth + 1), + HiveUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = HiveBinaryComparisonOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = HiveExpressionGenerator.HiveBinaryLogicalOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), + generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new HiveBinaryOperation(generateExpression(depth + 1), + generateExpression(depth + 1), + HiveExpressionGenerator.HiveBinaryArithmeticOperator.getRandom()); + case CAST: + // return new HiveCastOperation(generateExpression(depth + 1), + // HiveSchema.HiveCompositeDataType.getRandomWithoutNull()); + return new HiveCastOperation(generateExpression(depth + 1), HiveDataType.getRandomType()); + case FUNC: + HiveFunc func = HiveFunc.getRandom(); + return new HiveFunction<>(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new HiveBetweenOperation(generateExpression(depth + 1), + generateExpression(depth + 1), + generateExpression(depth + 1), + Randomly.getBoolean()); + case IN: + return new HiveInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), + Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new HiveCaseOperation(generateExpression(depth + 1), + generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), + generateExpression(depth + 1)); + default: + throw new AssertionError(expr); + } + } + + @Override + public HiveExpression generateConstant() { + ConstantType[] values = ConstantType.values(); + ConstantType constantType = Randomly.fromOptions(values); + switch (constantType) { + case STRING: + return HiveConstant.createStringConstant(globalState.getRandomly().getString()); + case INT: + return HiveConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DOUBLE: + return HiveConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BOOLEAN: + return HiveConstant.createBooleanConstant(Randomly.getBoolean()); + case NULL: + return HiveConstant.createNullConstant(); + default: + throw new AssertionError(constantType); + } + } + + @Override + protected HiveExpression generateColumn() { + HiveColumn column = Randomly.fromList(columns); + return new HiveColumnReference(column); + } + + @Override + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (HiveExpression curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new HiveOrderingTerm(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + } + + public enum HiveUnaryPrefixOperator implements Operator { + + // TODO: ~A (bitwise NOT) + NOT("NOT"), PLUS("+"), MINUS("-"); + + private String textRepr; + + HiveUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveUnaryPostfixOperator implements Operator { + + // TODO: A IS [NOT] (NULL|TRUE|FALSE)... + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + HiveUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveBinaryComparisonOperator implements Operator { + + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), + SMALLER_EQUALS("<="), NOT_EQUALS("!="), LIKE("LIKE"), + NOT_LIKE("NOT LIKE"), REGEXP("RLIKE"); + + private String textRepr; + + HiveBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveBinaryLogicalOperator implements Operator { + + AND("AND"), OR("OR"); + + private String textRepr; + + HiveBinaryLogicalOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveBinaryArithmeticOperator implements Operator { + + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), BITWISE_AND("&"), BITWISE_OR("|"), + BITWISE_XOR("^"); + + private String textRepr; + + HiveBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static HiveBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum HiveAggregateFunction { + COUNT(1), + SUM(1), + AVG(1), + MIN(1), + MAX(1), + VARIANCE(1), + VAR_SAMP(1), + STDDEV_POP(1), + STDDEV_SAMP(1), + COVAR_POP(2), + COVAR_SAMP(2), + CORR(2); + + private int nrArgs; + + HiveAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static HiveAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + // TODO: test all Hive default functions... + public enum HiveFunc { + + // mathematical functions + ROUND(2), + FLOOR(1); + + // collection functions + + // date functions + + // string functions + + + private int nrArgs; + private boolean isVariadic; + + HiveFunc(int nrArgs) { + this(nrArgs, false); + } + + HiveFunc(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static HiveFunc getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + + } +} diff --git a/src/sqlancer/hive/gen/HiveInsertGenerator.java b/src/sqlancer/hive/gen/HiveInsertGenerator.java new file mode 100644 index 000000000..963fafbce --- /dev/null +++ b/src/sqlancer/hive/gen/HiveInsertGenerator.java @@ -0,0 +1,52 @@ +package sqlancer.hive.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.HiveToStringVisitor; + +public class HiveInsertGenerator extends AbstractInsertGenerator { + + private final HiveGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + private final HiveExpressionGenerator gen; + + public HiveInsertGenerator(HiveGlobalState globalState) { + this.globalState = globalState; + this.gen = new HiveExpressionGenerator(globalState); + } + + public static SQLQueryAdapter getQuery(HiveGlobalState globalState) { + return new HiveInsertGenerator(globalState).generate(); + } + + @Override + protected void insertValue(HiveColumn column) { + sb.append(HiveToStringVisitor.asString(gen.generateConstant())); + } + + private SQLQueryAdapter generate() { + // Inserting values into tables from SQL. + sb.append("INSERT INTO "); + HiveTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + + // TODO: specify the inserted partition + + sb.append(" VALUES "); + + // Values must be provided by every column in the Hive table. + // A value is either null or any valid SQL literal. + List columns = table.getColumns(); + insertColumns(columns); + + HiveErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } +} diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java new file mode 100644 index 000000000..6f1a2121b --- /dev/null +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -0,0 +1,125 @@ +package sqlancer.hive.gen; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema; +import sqlancer.hive.HiveToStringVisitor; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveDataType; +import sqlancer.hive.HiveSchema.HiveTable; + +import java.util.ArrayList; +import java.util.List; + +public class HiveTableGenerator { + + // TODO: support various file formats? e.g. JSONFILE, SEQUENCEFILE, TEXTFILE, RCFILE, ORC, PARQUET, AVRO. + + private enum ColumnConstraints { + PRIMARY_KEY_DISABLE, UNIQUE_DISABLE, NOT_NULL, DEFAULT, CHECK + // ENABLE_OR_DISABLE, NOVALIDATE, RELY_OR_NORELY + } + + private final HiveGlobalState globalState; + private final String tableName; + private final boolean allowPrimaryKey = Randomly.getBoolean(); + private final StringBuilder sb = new StringBuilder(); + private final HiveExpressionGenerator gen; + private final HiveTable table; + private final List columnsToBeAdded = new ArrayList<>(); + private boolean setPrimaryKey = false; + + public HiveTableGenerator(HiveGlobalState globalState, String tableName) { + this.tableName = tableName; + this.globalState = globalState; + this.table = new HiveTable(tableName, columnsToBeAdded, false); + this.gen = new HiveExpressionGenerator(globalState).setColumns(columnsToBeAdded); + } + + public static SQLQueryAdapter generate(HiveGlobalState globalState, String tableName) { + HiveTableGenerator generator = new HiveTableGenerator(globalState, tableName); + return generator.create(); + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + + sb.append("CREATE "); + sb.append("TABLE "); + sb.append(globalState.getDatabaseName()); + sb.append("."); + sb.append(tableName); + sb.append(" ("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(i, errors); + } + sb.append(")"); + + // TODO: implement PARTITION BY clause + // TODO: implement CLUSTERED BY, SKEWED BY clauses + // TODO: implement ROW FORMAT and STORED AS clauses + // TODO: randomly add some predefined TABLEPROPERTIES + // TODO: implement CTAS (AS clause) + + HiveErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + + private void appendColumn(int columnId, ExpectedErrors errors) { + String columnName = DBMSCommon.createColumnName(columnId); + sb.append(columnName); + sb.append(" "); + HiveDataType randType = HiveSchema.HiveDataType.getRandomType(); + sb.append(randType); + columnsToBeAdded.add(new HiveColumn(columnName, table, randType)); + appendColumnConstraint(randType, errors); + } + + private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { + /* column_constraint_specification: + : [ PRIMARY KEY|UNIQUE|NOT NULL|DEFAULT [default_value]|CHECK [check_expression] ENABLE|DISABLE NOVALIDATE RELY/NORELY ] + */ + if (Randomly.getBoolean()) { + // no column constraint + return; + } + + ColumnConstraints constraint = Randomly.fromOptions(ColumnConstraints.values()); + switch (constraint) { + case PRIMARY_KEY_DISABLE: + if (allowPrimaryKey && !setPrimaryKey) { + sb.append(" PRIMARY KEY DISABLE"); + setPrimaryKey = true; + } + break; + case UNIQUE_DISABLE: + sb.append(" UNIQUE DISABLE"); + break; + case NOT_NULL: + sb.append(" NOT NULL"); + break; + case DEFAULT: + // TODO: add errors + sb.append(" DEFAULT ("); + sb.append(HiveToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + case CHECK: + errors.add("Invalid Constraint syntax"); + sb.append(" CHECK ("); + sb.append(HiveToStringVisitor.asString(gen.generateExpression())); + sb.append(")"); + break; + default: + throw new AssertionError(constraint); + } + } + + +} diff --git a/src/sqlancer/hive/oracle/HiveTLPBase.java b/src/sqlancer/hive/oracle/HiveTLPBase.java new file mode 100644 index 000000000..cff69d06b --- /dev/null +++ b/src/sqlancer/hive/oracle/HiveTLPBase.java @@ -0,0 +1,68 @@ +package sqlancer.hive.oracle; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.gen.ExpressionGenerator; +import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveSchema; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.HiveSchema.HiveTables; +import sqlancer.hive.ast.HiveColumnReference; +import sqlancer.hive.ast.HiveExpression; +import sqlancer.hive.ast.HiveSelect; +import sqlancer.hive.ast.HiveTableReference; +import sqlancer.hive.gen.HiveExpressionGenerator; + +public class HiveTLPBase extends TernaryLogicPartitioningOracleBase + implements TestOracle { + + HiveSchema schema; + HiveTables targetTables; + HiveExpressionGenerator gen; + HiveSelect select; + + public HiveTLPBase(HiveGlobalState state) { + super(state); + HiveErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws Exception { + schema = state.getSchema(); + targetTables = schema.getRandomTableNonEmptyTables(); + gen = new HiveExpressionGenerator(state).setColumns(targetTables.getColumns()); + initializeTernaryPredicateVariants(); + select = new HiveSelect(); + select.setFetchColumns(generateFetchColumns()); + List tables = targetTables.getTables(); + List tableList = tables.stream().map(t -> new HiveTableReference(t)) + .collect(Collectors.toList()); + // List joins = HiveJoin.getJoins(tableList, state); + // select.setJoinList(joins.stream().collect(Collectors.toList())); + select.setFromList(tableList.stream().collect(Collectors.toList())); + select.setWhereClause(null); + } + + List generateFetchColumns() { + List columns = new ArrayList<>(); + if (Randomly.getBoolean()) { + columns.add(new HiveColumnReference(new HiveColumn("*", null, null))); + } else { + columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new HiveColumnReference(c)) + .collect(ArrayList::new, ArrayList::add, ArrayList::addAll); + } + return columns; + } + + @Override + protected ExpressionGenerator getGen() { + return gen; + } +} diff --git a/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java b/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java new file mode 100644 index 000000000..d59867058 --- /dev/null +++ b/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java @@ -0,0 +1,42 @@ +package sqlancer.hive.oracle; + +import java.util.List; +import java.util.ArrayList; + +import sqlancer.ComparatorHelper; +import sqlancer.Randomly; +import sqlancer.hive.HiveErrors; +import sqlancer.hive.HiveGlobalState; +import sqlancer.hive.HiveToStringVisitor; + +public class HiveTLPWhereOracle extends HiveTLPBase { + + public HiveTLPWhereOracle(HiveGlobalState state) { + super(state); + HiveErrors.addExpressionErrors(errors); + } + + @Override + public void check() throws Exception { + super.check(); + String originalQueryString = HiveToStringVisitor.asString(select); + List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + + boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); + if (orderBy) { + select.setOrderByClauses(gen.generateOrderBys()); + } + + select.setWhereClause(predicate); + String firstQueryString = HiveToStringVisitor.asString(select); + select.setWhereClause(negatedPredicate); + String secondQueryString = HiveToStringVisitor.asString(select); + select.setWhereClause(isNullPredicate); + String thirdQueryString = HiveToStringVisitor.asString(select); + List combinedString = new ArrayList<>(); + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, + state, ComparatorHelper::canonicalizeResultValue); + } +} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index 92de68fe9..f5aeefa12 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -10,6 +10,7 @@ public class TestConfig { public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; + public static final String HIVE_ENV = "HIVE_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; public static final String MYSQL_ENV = "MYSQL_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestHiveTLP.java b/test/sqlancer/dbms/TestHiveTLP.java new file mode 100644 index 000000000..05a4bc20d --- /dev/null +++ b/test/sqlancer/dbms/TestHiveTLP.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestHiveTLP { + + @Test + public void testHiveTLPWhere() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.HIVE_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", + "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", + "--num-queries", TestConfig.NUM_QUERIES, "hive", "--oracle", "TLPWhere" })); + } +} \ No newline at end of file From 6e8669857eff0f2d592a831801ad4e1fcbe68c1f Mon Sep 17 00:00:00 2001 From: liyxiris Date: Wed, 9 Apr 2025 19:15:21 +0800 Subject: [PATCH 664/856] [Hive] Remove redundant lines --- src/sqlancer/hive/HiveSchema.java | 9 +------- src/sqlancer/hive/ast/HiveConstant.java | 23 ------------------- .../hive/gen/HiveExpressionGenerator.java | 16 ++++--------- src/sqlancer/hive/gen/HiveTableGenerator.java | 4 +--- 4 files changed, 7 insertions(+), 45 deletions(-) diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java index 018293093..8ae33bf99 100644 --- a/src/sqlancer/hive/HiveSchema.java +++ b/src/sqlancer/hive/HiveSchema.java @@ -23,7 +23,7 @@ public class HiveSchema extends AbstractSchema { public enum HiveDataType { // TODO: support more types, e.g. TIMESTAMP, DATE, VARCHAR, CHAR, BINARY, ARRAY, MAP, STRUCT, UNIONTYPE... - INT, FLOAT, DECIMAL, STRING, BOOLEAN; + STRING, INT, DOUBLE, BOOLEAN; public static HiveDataType getRandomType() { return Randomly.fromList(Arrays.asList(values())); @@ -56,13 +56,9 @@ public HiveSchema(List databaseTables) { } public static HiveSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { - // TODO: discover built-in tables List databaseTables = new ArrayList<>(); List tableNames = getTableNames(con); for (String tableName : tableNames) { - if (DBMSCommon.matchesIndexName(tableName)) { - continue; // TODO: unexpected? - } List databaseColumns = getTableColumns(con, tableName); boolean isView = tableName.startsWith("v"); HiveTable t = new HiveTable(tableName, databaseColumns, isView); @@ -102,9 +98,6 @@ private static List getTableColumns(SQLConnection con, String tableN } private static HiveDataType getColumnType(String typeString) { - if (typeString.startsWith("DECIMAL")) { - return HiveDataType.DECIMAL; - } return HiveDataType.valueOf(typeString.toUpperCase()); } diff --git a/src/sqlancer/hive/ast/HiveConstant.java b/src/sqlancer/hive/ast/HiveConstant.java index ed3ed591d..7f89997d2 100644 --- a/src/sqlancer/hive/ast/HiveConstant.java +++ b/src/sqlancer/hive/ast/HiveConstant.java @@ -41,29 +41,6 @@ public String toString() { } } - // public static class HiveFloatConstant extends HiveConstant { - - // private final float value; - - // public HiveFloatConstant(float value) { - // this.value = value; - // } - - // public float getValue() { - // return value; - // } - - // @Override - // public String toString() { - // if (value == Float.POSITIVE_INFINITY) { - // return "'+Inf'"; - // } else if (value == Float.NEGATIVE_INFINITY) { - // return "'-Inf'"; - // } - // return String.valueOf(value); - // } - // } - public static class HiveDoubleConstant extends HiveConstant { private final double value; diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index d8949a72c..ca2e53929 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -33,11 +33,6 @@ private enum Expression { CAST, FUNC, BETWEEN, IN, CASE; } - private enum ConstantType { - // TODO: add DECIMAL, DATE, TIMESTAMP, BINARY,... - STRING, INT, DOUBLE, BOOLEAN, NULL - } - public HiveExpressionGenerator(HiveGlobalState globalState) { this.globalState = globalState; } @@ -93,8 +88,6 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr generateExpression(depth + 1), HiveExpressionGenerator.HiveBinaryArithmeticOperator.getRandom()); case CAST: - // return new HiveCastOperation(generateExpression(depth + 1), - // HiveSchema.HiveCompositeDataType.getRandomWithoutNull()); return new HiveCastOperation(generateExpression(depth + 1), HiveDataType.getRandomType()); case FUNC: HiveFunc func = HiveFunc.getRandom(); @@ -121,8 +114,11 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr @Override public HiveExpression generateConstant() { - ConstantType[] values = ConstantType.values(); - ConstantType constantType = Randomly.fromOptions(values); + if (Randomly.getBooleanWithRatherLowProbability()) { + return HiveConstant.createNullConstant(); + } + HiveDataType[] values = HiveDataType.values(); + HiveDataType constantType = Randomly.fromOptions(values); switch (constantType) { case STRING: return HiveConstant.createStringConstant(globalState.getRandomly().getString()); @@ -132,8 +128,6 @@ public HiveExpression generateConstant() { return HiveConstant.createDoubleConstant(globalState.getRandomly().getDouble()); case BOOLEAN: return HiveConstant.createBooleanConstant(Randomly.getBoolean()); - case NULL: - return HiveConstant.createNullConstant(); default: throw new AssertionError(constantType); } diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java index 6f1a2121b..2d91325bf 100644 --- a/src/sqlancer/hive/gen/HiveTableGenerator.java +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -48,8 +48,7 @@ public static SQLQueryAdapter generate(HiveGlobalState globalState, String table private SQLQueryAdapter create() { ExpectedErrors errors = new ExpectedErrors(); - sb.append("CREATE "); - sb.append("TABLE "); + sb.append("CREATE TABLE "); sb.append(globalState.getDatabaseName()); sb.append("."); sb.append(tableName); @@ -106,7 +105,6 @@ private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { sb.append(" NOT NULL"); break; case DEFAULT: - // TODO: add errors sb.append(" DEFAULT ("); sb.append(HiveToStringVisitor.asString(gen.generateConstant())); sb.append(")"); From ab731ec7f06554bfc365197bae3ac0b156196f18 Mon Sep 17 00:00:00 2001 From: liyxiris Date: Sun, 13 Apr 2025 20:37:29 +0800 Subject: [PATCH 665/856] [Hive] refactor Hive TLP oracle using generic approaches --- src/sqlancer/hive/HiveOptions.java | 14 +++- src/sqlancer/hive/HiveSchema.java | 1 - src/sqlancer/hive/ast/HiveSelect.java | 12 ++-- .../hive/gen/HiveExpressionGenerator.java | 49 ++++++++++++- src/sqlancer/hive/oracle/HiveTLPBase.java | 68 ------------------- .../hive/oracle/HiveTLPWhereOracle.java | 42 ------------ 6 files changed, 65 insertions(+), 121 deletions(-) delete mode 100644 src/sqlancer/hive/oracle/HiveTLPBase.java delete mode 100644 src/sqlancer/hive/oracle/HiveTLPWhereOracle.java diff --git a/src/sqlancer/hive/HiveOptions.java b/src/sqlancer/hive/HiveOptions.java index fff2d74b1..a8c049756 100644 --- a/src/sqlancer/hive/HiveOptions.java +++ b/src/sqlancer/hive/HiveOptions.java @@ -2,11 +2,15 @@ import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; + +import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; import sqlancer.DBMSSpecificOptions; -import sqlancer.hive.oracle.HiveTLPWhereOracle; +import sqlancer.hive.gen.HiveExpressionGenerator; import sqlancer.OracleFactory; +import java.sql.SQLException; import java.util.Arrays; import java.util.List; @@ -22,8 +26,12 @@ public class HiveOptions implements DBMSSpecificOptions { TLPWhere { @Override - public TestOracle create(HiveGlobalState globalState) { - return new HiveTLPWhereOracle(globalState); + public TestOracle create(HiveGlobalState globalState) throws SQLException { + HiveExpressionGenerator gen = new HiveExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors() + .with(HiveErrors.getExpressionErrors()).build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); } }; } diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java index 8ae33bf99..fcc17c77c 100644 --- a/src/sqlancer/hive/HiveSchema.java +++ b/src/sqlancer/hive/HiveSchema.java @@ -2,7 +2,6 @@ import sqlancer.Randomly; import sqlancer.SQLConnection; -import sqlancer.common.DBMSCommon; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTableColumn; diff --git a/src/sqlancer/hive/ast/HiveSelect.java b/src/sqlancer/hive/ast/HiveSelect.java index ffd5e8fbd..4853ef892 100644 --- a/src/sqlancer/hive/ast/HiveSelect.java +++ b/src/sqlancer/hive/ast/HiveSelect.java @@ -4,6 +4,7 @@ import sqlancer.common.ast.SelectBase; import sqlancer.common.ast.newast.Select; +import sqlancer.hive.HiveToStringVisitor; import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveTable; @@ -22,20 +23,19 @@ public boolean isDistinct() { @Override public void setJoinClauses(List joinStatements) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'setJoinClauses'"); + List expressions = joinStatements.stream().map(e -> (HiveExpression) e) + .collect(java.util.stream.Collectors.toList()); + setJoinList(expressions); } @Override public List getJoinClauses() { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'getJoinClauses'"); + return getJoinList().stream().map(e -> (HiveJoin) e).collect(java.util.stream.Collectors.toList()); } @Override public String asString() { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'asString'"); + return HiveToStringVisitor.asString(this); } } diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index ca2e53929..3e35544c5 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -2,7 +2,9 @@ import sqlancer.Randomly; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; import sqlancer.hive.HiveGlobalState; import sqlancer.hive.HiveSchema.*; import sqlancer.hive.ast.HiveBetweenOperation; @@ -14,7 +16,10 @@ import sqlancer.hive.ast.HiveExpression; import sqlancer.hive.ast.HiveFunction; import sqlancer.hive.ast.HiveInOperation; +import sqlancer.hive.ast.HiveJoin; import sqlancer.hive.ast.HiveOrderingTerm; +import sqlancer.hive.ast.HiveSelect; +import sqlancer.hive.ast.HiveTableReference; import sqlancer.hive.ast.HiveUnaryPrefixOperation; import sqlancer.hive.ast.HiveUnaryPostfixOperation; import sqlancer.common.ast.BinaryOperatorNode.Operator; @@ -22,10 +27,13 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; -public class HiveExpressionGenerator extends UntypedExpressionGenerator { +public class HiveExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { private final HiveGlobalState globalState; + private List tables; private enum Expression { // TODO: add or delete expressions. @@ -152,6 +160,45 @@ public List generateOrderBys() { return newExpr; } + + @Override + public HiveExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + + return this; + } + + @Override + public HiveExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public HiveSelect generateSelect() { + return new HiveSelect(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new HiveTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean allowAggregates) { + if (Randomly.getBoolean()) { + return List.of(new HiveColumnReference(new HiveColumn("*", null, null))); + } + return Randomly.nonEmptySubset(columns).stream() + .map(c -> new HiveColumnReference(c)) + .collect(Collectors.toList()); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + public enum HiveUnaryPrefixOperator implements Operator { // TODO: ~A (bitwise NOT) diff --git a/src/sqlancer/hive/oracle/HiveTLPBase.java b/src/sqlancer/hive/oracle/HiveTLPBase.java deleted file mode 100644 index cff69d06b..000000000 --- a/src/sqlancer/hive/oracle/HiveTLPBase.java +++ /dev/null @@ -1,68 +0,0 @@ -package sqlancer.hive.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; -import sqlancer.hive.HiveErrors; -import sqlancer.hive.HiveGlobalState; -import sqlancer.hive.HiveSchema; -import sqlancer.hive.HiveSchema.HiveColumn; -import sqlancer.hive.HiveSchema.HiveTable; -import sqlancer.hive.HiveSchema.HiveTables; -import sqlancer.hive.ast.HiveColumnReference; -import sqlancer.hive.ast.HiveExpression; -import sqlancer.hive.ast.HiveSelect; -import sqlancer.hive.ast.HiveTableReference; -import sqlancer.hive.gen.HiveExpressionGenerator; - -public class HiveTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - HiveSchema schema; - HiveTables targetTables; - HiveExpressionGenerator gen; - HiveSelect select; - - public HiveTLPBase(HiveGlobalState state) { - super(state); - HiveErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws Exception { - schema = state.getSchema(); - targetTables = schema.getRandomTableNonEmptyTables(); - gen = new HiveExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new HiveSelect(); - select.setFetchColumns(generateFetchColumns()); - List tables = targetTables.getTables(); - List tableList = tables.stream().map(t -> new HiveTableReference(t)) - .collect(Collectors.toList()); - // List joins = HiveJoin.getJoins(tableList, state); - // select.setJoinList(joins.stream().collect(Collectors.toList())); - select.setFromList(tableList.stream().collect(Collectors.toList())); - select.setWhereClause(null); - } - - List generateFetchColumns() { - List columns = new ArrayList<>(); - if (Randomly.getBoolean()) { - columns.add(new HiveColumnReference(new HiveColumn("*", null, null))); - } else { - columns = Randomly.nonEmptySubset(targetTables.getColumns()).stream().map(c -> new HiveColumnReference(c)) - .collect(ArrayList::new, ArrayList::add, ArrayList::addAll); - } - return columns; - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } -} diff --git a/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java b/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java deleted file mode 100644 index d59867058..000000000 --- a/src/sqlancer/hive/oracle/HiveTLPWhereOracle.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.hive.oracle; - -import java.util.List; -import java.util.ArrayList; - -import sqlancer.ComparatorHelper; -import sqlancer.Randomly; -import sqlancer.hive.HiveErrors; -import sqlancer.hive.HiveGlobalState; -import sqlancer.hive.HiveToStringVisitor; - -public class HiveTLPWhereOracle extends HiveTLPBase { - - public HiveTLPWhereOracle(HiveGlobalState state) { - super(state); - HiveErrors.addExpressionErrors(errors); - } - - @Override - public void check() throws Exception { - super.check(); - String originalQueryString = HiveToStringVisitor.asString(select); - List resultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); - - boolean orderBy = Randomly.getBooleanWithRatherLowProbability(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBys()); - } - - select.setWhereClause(predicate); - String firstQueryString = HiveToStringVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = HiveToStringVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = HiveToStringVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state, ComparatorHelper::canonicalizeResultValue); - } -} From b9c13c35ff4a29a78bfd7dfdd8c75e4cc8cde833 Mon Sep 17 00:00:00 2001 From: liyxiris Date: Wed, 16 Apr 2025 18:14:55 +0800 Subject: [PATCH 666/856] [Hive] upgrade Hive dependencies & remove expected 'Invalid Constraint syntax' error --- pom.xml | 9 +++++++-- src/sqlancer/hive/gen/HiveTableGenerator.java | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 1894cae67..bf30df4ce 100644 --- a/pom.xml +++ b/pom.xml @@ -381,12 +381,17 @@ org.apache.hive hive-jdbc - 3.1.3 + 4.0.1 org.apache.hive hive-serde - 3.1.3 + 4.0.1 + + + org.apache.hive + hive-cli + 4.0.1 diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java index 2d91325bf..43192a5be 100644 --- a/src/sqlancer/hive/gen/HiveTableGenerator.java +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -109,7 +109,6 @@ private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { sb.append(HiveToStringVisitor.asString(gen.generateConstant())); sb.append(")"); case CHECK: - errors.add("Invalid Constraint syntax"); sb.append(" CHECK ("); sb.append(HiveToStringVisitor.asString(gen.generateExpression())); sb.append(")"); From d45cd022e8f710ace2105aa79bc4eb949774bfc0 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 17 Apr 2025 13:20:02 +0800 Subject: [PATCH 667/856] modify variabels name to follow Java naming convertions, fix an error in SQLite3ToStringVisitor, add unit tests for AbstractTables and CODDTest for SQLite --- src/sqlancer/sqlite3/SQLite3Options.java | 14 ++++---- .../sqlite3/SQLite3ToStringVisitor.java | 2 +- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 2 +- test/sqlancer/TestCommonSchema.java | 34 +++++++++++++++++++ test/sqlancer/dbms/TestSQLiteCODDTest.java | 18 ++++++++++ 5 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 test/sqlancer/dbms/TestSQLiteCODDTest.java diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index 6a5188337..fce9f04ba 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -79,20 +79,20 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") public int maxNumIndexes = 20; - public enum coddtest_model { - random, expression, subquery; + public enum CODDTestModel { + RANDOM, EXPRESSION, SUBQUERY; public boolean isRandom() { - return this == random; + return this == RANDOM; } public boolean isExpression() { - return this == expression; + return this == EXPRESSION; } public boolean isSubquery() { - return this == subquery; + return this == SUBQUERY; } } - @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") - public coddtest_model coddTestModel = coddtest_model.random; + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on EXPRESSION, SUBQUERY, or RANDOM") + public CODDTestModel coddTestModel = CODDTestModel.RANDOM; @Override public List getTestOracleFactory() { diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 4b504a4be..dd51baafd 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -552,7 +552,7 @@ public void visit(SQLite3Values values) { sb.append(vs.get(name).get(i).getDataType().toString()); sb.append("))"); } - isFirstColumn = true; + isFirstColumn = false; } sb.append(")"); if (i < size - 1) { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 049ca7fe4..8198eadb6 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -936,7 +936,7 @@ public boolean useSubquery() { } else if (this.state.getDbmsSpecificOptions().coddTestModel.isSubquery()) { return true; } else { - System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); + System.out.printf("Wrong option of --coddtest-model, should be one of: RANDOM, EXPRESSION, SUBQUERY"); System.exit(1); return false; } diff --git a/test/sqlancer/TestCommonSchema.java b/test/sqlancer/TestCommonSchema.java index 80a57dd8c..c77892124 100644 --- a/test/sqlancer/TestCommonSchema.java +++ b/test/sqlancer/TestCommonSchema.java @@ -4,6 +4,7 @@ import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; import sqlancer.common.schema.TableIndex; import java.util.*; @@ -35,6 +36,12 @@ static class TestSchema extends AbstractSchema, TestTable> } } + static class TestTables extends AbstractTables { + TestTables(List tables) { + super(tables); + } + } + static class TestIndex extends TableIndex { TestIndex(String name) { super(name); @@ -55,6 +62,10 @@ private TestSchema createTestSchema(TestTable... tables) { return new TestSchema(Arrays.asList(tables)); } + private TestTables createTestTables(TestTable... tables) { + return new TestTables(new ArrayList(Arrays.asList(tables))); + } + @Test void testColumnManagement() { TestTable table = createTestTable("products", Collections.emptyList(), false, "sku", "price"); @@ -149,4 +160,27 @@ void testBoundaryConditions() { assertEquals("orphan", col2.getFullQualifiedName()); assertNull(col2.getTable()); } + + @Test + void testTablesManagement() { + TestTable table1 = createTestTable("t1", Collections.emptyList(), false, "col1"); + TestTable table2 = createTestTable("t2", Collections.emptyList(), false, "col2"); + TestTable table3 = createTestTable("t3", Collections.emptyList(), false, "col3"); + + TestTables tables = createTestTables(table1, table2, table3); + assertEquals(3, tables.getSize(), "Should detect 3 tables"); + assertEquals(3, tables.getColumns().size(), "Should detect 3 columns"); + assertTrue(tables.isContained(table3), "Table3 shoule be contained"); + + TestTable table4 = createTestTable("t4", Collections.emptyList(), false, "col4"); + tables.addTable(table4); + assertEquals(4, tables.getSize(), "Should detect 4 tables"); + assertEquals(4, tables.getColumns().size(), "Should detect 4 columns"); + assertTrue(tables.isContained(table4), "Table4 should be contained"); + + tables.removeTable(table4); + assertEquals(3, tables.getSize(), "Should detect 3 tables"); + assertEquals(3, tables.getColumns().size(), "Should detect 3 columns"); + assertTrue(!tables.isContained(table4), "Table4 should not be contained"); + } } diff --git a/test/sqlancer/dbms/TestSQLiteCODDTest.java b/test/sqlancer/dbms/TestSQLiteCODDTest.java new file mode 100644 index 000000000..6697c09c5 --- /dev/null +++ b/test/sqlancer/dbms/TestSQLiteCODDTest.java @@ -0,0 +1,18 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSQLiteCODDTest { + + @Test + public void testSqliteCODDTest() { + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", + "CODDTest" })); + } +} From c00a19bb52b21247ae2ec862b3421195f87a0306 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Tue, 22 Apr 2025 14:20:26 +0800 Subject: [PATCH 668/856] format the CODDTest for SQLite3 code --- src/sqlancer/common/oracle/CODDTestBase.java | 2 +- .../common/schema/AbstractTables.java | 1 - .../sqlite3/SQLite3ExpectedValueVisitor.java | 4 +- src/sqlancer/sqlite3/SQLite3Options.java | 4 + .../sqlite3/SQLite3ToStringVisitor.java | 10 +- src/sqlancer/sqlite3/SQLite3Visitor.java | 3 +- .../sqlite3/ast/SQLite3Expression.java | 32 +-- src/sqlancer/sqlite3/ast/SQLite3Select.java | 6 +- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 248 +++++++++--------- .../sqlite3/schema/SQLite3DataType.java | 27 +- test/sqlancer/TestCommonSchema.java | 2 +- test/sqlancer/dbms/TestSQLiteCODDTest.java | 6 +- 12 files changed, 178 insertions(+), 167 deletions(-) diff --git a/src/sqlancer/common/oracle/CODDTestBase.java b/src/sqlancer/common/oracle/CODDTestBase.java index 162a8eaed..639a4f077 100644 --- a/src/sqlancer/common/oracle/CODDTestBase.java +++ b/src/sqlancer/common/oracle/CODDTestBase.java @@ -22,4 +22,4 @@ public CODDTestBase(S state) { this.logger = state.getLogger(); this.options = state.getOptions(); } -} \ No newline at end of file +} diff --git a/src/sqlancer/common/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java index 67da6364e..ff20fde20 100644 --- a/src/sqlancer/common/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -34,7 +34,6 @@ public String columnNamesAsString(Function function) { return getColumns().stream().map(function).collect(Collectors.joining(", ")); } - public void addTable(T table) { if (!this.tables.contains(table)) { this.tables.add(table); diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 594fe5c92..7ca82ec02 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -22,9 +22,9 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; @@ -310,7 +310,7 @@ public void visit(SQLite3SetClause set) { visit(set.getRight()); } - @Override + @Override public void visit(SQLite3Alias alias) { print(alias); print(alias.getOriginalExpression()); diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index fce9f04ba..e9e34892e 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -81,16 +81,20 @@ public class SQLite3Options implements DBMSSpecificOptions public enum CODDTestModel { RANDOM, EXPRESSION, SUBQUERY; + public boolean isRandom() { return this == RANDOM; } + public boolean isExpression() { return this == EXPRESSION; } + public boolean isSubquery() { return this == SUBQUERY; } } + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on EXPRESSION, SUBQUERY, or RANDOM") public CODDTestModel coddTestModel = CODDTestModel.RANDOM; diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index dd51baafd..8fdbf4438 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -29,9 +29,9 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; @@ -317,7 +317,7 @@ public void visit(InOperation op) { sb.append(")"); } } - + sb.append(")"); } @@ -534,7 +534,7 @@ public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { public void visit(SQLite3Values values) { Map> vs = values.getValues(); int size = vs.get(vs.keySet().iterator().next()).size(); - List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + List columnNames = values.getColumns().stream().map(c -> c.getName()).collect(Collectors.toList()); sb.append("(VALUES "); for (int i = 0; i < size; i++) { sb.append("("); @@ -586,7 +586,7 @@ public void visit(SQLite3ResultMap tableSummary) { if (size == 0) { throw new AssertionError("The result of the expression must not be empty."); } - List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + List columnNames = values.getColumns().stream().map(c -> c.getName()).collect(Collectors.toList()); sb.append(" CASE "); for (int i = 0; i < size; i++) { sb.append("WHEN "); @@ -608,4 +608,4 @@ public void visit(SQLite3ResultMap tableSummary) { } sb.append("END "); } -} \ No newline at end of file +} diff --git a/src/sqlancer/sqlite3/SQLite3Visitor.java b/src/sqlancer/sqlite3/SQLite3Visitor.java index 5e1e5dbf6..02ac2b4d7 100644 --- a/src/sqlancer/sqlite3/SQLite3Visitor.java +++ b/src/sqlancer/sqlite3/SQLite3Visitor.java @@ -21,9 +21,9 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; @@ -137,7 +137,6 @@ default void visit(SQLite3PostfixUnaryOperation exp) { void visit(SQLite3WindowFunctionFrameSpecBetween between); - void visit(SQLite3Alias alias); void visit(SQLite3WithClause withClause); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 681566c03..ad0f64ad6 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1,8 +1,8 @@ package sqlancer.sqlite3.ast; import java.util.List; -import java.util.Optional; import java.util.Map; +import java.util.Optional; import sqlancer.IgnoreMeException; import sqlancer.LikeImplementationHelper; @@ -114,7 +114,7 @@ public SQLite3CollateSequence getImplicitCollateSequence() { public static class SQLite3Exist extends SQLite3Expression { private final SQLite3Expression select; - private boolean negated = false; + private boolean negated; public SQLite3Exist(SQLite3Expression select, boolean negated) { this.select = select; @@ -1565,7 +1565,7 @@ public boolean omitBracketsWhenPrinting() { public static class SQLite3WithClause extends SQLite3Expression { - private SQLite3Expression left; + private final SQLite3Expression left; private SQLite3Expression right; public SQLite3WithClause(SQLite3Expression left, SQLite3Expression right) { @@ -1593,23 +1593,23 @@ public SQLite3CollateSequence getExplicitCollateSequence() { public static class SQLite3Alias extends SQLite3Expression { - private SQLite3Expression originalExpression; - private SQLite3Expression aliasExpression; - + private final SQLite3Expression originalExpression; + private final SQLite3Expression aliasExpression; + public SQLite3Alias(SQLite3Expression originalExpression, SQLite3Expression aliasExpression) { this.originalExpression = originalExpression; this.aliasExpression = aliasExpression; } - + @Override public SQLite3CollateSequence getExplicitCollateSequence() { return null; } - + public SQLite3Expression getOriginalExpression() { return originalExpression; } - + public SQLite3Expression getAliasExpression() { return aliasExpression; } @@ -1673,8 +1673,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } } - - // The ExpressionBag is not a built-in SQL feature, + // The ExpressionBag is not a built-in SQL feature, // but rather a utility class used in CODDTest's oracle construction // to substitute expressions with their corresponding constant values. public static class SQLite3ExpressionBag extends SQLite3Expression { @@ -1700,7 +1699,7 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } public static class SQLite3Typeof extends SQLite3Expression { - private SQLite3Expression innerExpr; + private final SQLite3Expression innerExpr; public SQLite3Typeof(SQLite3Expression innerExpr) { this.innerExpr = innerExpr; @@ -1714,7 +1713,7 @@ public SQLite3Expression getInnerExpr() { public SQLite3CollateSequence getExplicitCollateSequence() { return null; } - + } public static class SQLite3ResultMap extends SQLite3Expression { @@ -1723,7 +1722,8 @@ public static class SQLite3ResultMap extends SQLite3Expression { private final List summary; private final SQLite3DataType summaryDataType; - public SQLite3ResultMap(SQLite3Values values, List columns, List summary, SQLite3DataType summaryDataType) { + public SQLite3ResultMap(SQLite3Values values, List columns, List summary, + SQLite3DataType summaryDataType) { this.values = values; this.columns = columns; this.summary = summary; @@ -1755,6 +1755,6 @@ public SQLite3DataType getSummaryDataType() { public SQLite3CollateSequence getExplicitCollateSequence() { return null; } - + } -} \ No newline at end of file +} diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index cf35b124a..176057e8a 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -25,7 +25,7 @@ public class SQLite3Select extends SQLite3Expression private List fetchColumns = Collections.emptyList(); private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; - private SQLite3WithClause withClause = null; + private SQLite3WithClause withClause; public SQLite3Select() { } @@ -178,14 +178,14 @@ public SQLite3Expression getWithClause() { } // This method is used in CODDTest to test subquery by replacing a table name - // in the SELECT clause with a derived table expression. + // in the SELECT clause with a derived table expression. public void replaceFromTable(String tableName, SQLite3Expression newFromExpression) { int replaceIdx = -1; for (int i = 0; i < fromList.size(); ++i) { SQLite3Expression f = fromList.get(i); if (f instanceof SQLite3TableReference) { SQLite3TableReference tableRef = (SQLite3TableReference) f; - if (tableRef.getTable().getName() == tableName) { + if (tableRef.getTable().getName().equals(tableName)) { replaceIdx = i; } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 8198eadb6..c262243a9 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -31,20 +31,20 @@ import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; -import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; import sqlancer.sqlite3.ast.SQLite3Select; import sqlancer.sqlite3.gen.SQLite3Common; @@ -61,13 +61,13 @@ public class SQLite3CODDTestOracle extends CODDTestBase impl private SQLite3ExpressionGenerator gen; private Reproducer reproducer; - private String tempTableName = "temp_table"; + private static final String TEMP_TABLE_NAME = "temp_table"; private SQLite3Expression foldedExpr; private SQLite3Expression constantResOfFoldedExpr; private List tablesFromOuterContext = new ArrayList<>(); - private List joinsInExpr = null; + private List joinsInExpr; Map> auxiliaryQueryResult = new HashMap<>(); Map> selectResult = new HashMap<>(); @@ -118,15 +118,14 @@ public void check() throws SQLException { auxiliaryQueryResult.putAll(selectResult); } - SQLite3Select originalQuery = null; - + Map> foldedResult = new HashMap<>(); Map> originalResult = new HashMap<>(); // dependent expression - if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) { // original query SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(this.foldedExpr); originalQuery = this.genSelectExpression(null, specificCondition); @@ -137,69 +136,71 @@ public void check() throws SQLException { specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } - // independent expression - // empty result, put the inner query in (NOT) EXIST - else if (auxiliaryQueryResult.size() == 0 || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).size() == 0) { - boolean isNegated = Randomly.getBoolean() ? false : true; + } else if (auxiliaryQueryResult.isEmpty() + || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).isEmpty()) { + // independent expression + // empty result, put the inner query in (NOT) EXIST + boolean isNegated = !Randomly.getBoolean(); // original query SQLite3Exist existExpr = new SQLite3Exist(new SQLite3Select(auxiliaryQuery), isNegated); SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(existExpr); - + originalQuery = this.genSelectExpression(null, specificCondition); originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); - + // folded query SQLite3Expression equivalentExpr = isNegated ? SQLite3Constant.createTrue() : SQLite3Constant.createFalse(); specificCondition.updateInnerExpr(equivalentExpr); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } - // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant - else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 && Randomly.getBoolean()) { + } else if (auxiliaryQueryResult.size() == 1 + && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 + && Randomly.getBoolean()) { + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant // original query SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(auxiliaryQuery); originalQuery = this.genSelectExpression(null, specificCondition); originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); - + // folded query - SQLite3Expression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); - specificCondition.updateInnerExpr(equivalentExpr);; + SQLite3Expression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]) + .get(0); + specificCondition.updateInnerExpr(equivalentExpr); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } - // one column - else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability() && testInOperator()) { + } else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability() + && enableInOperator()) { + // one column // original query List columns = s.getRandomTableNonEmptyTables().getColumns(); SQLite3ColumnName selectedColumn = new SQLite3ColumnName(Randomly.fromList(columns), null); SQLite3Table selectedTable = selectedColumn.getColumn().getTable(); - InOperation INOperation = new InOperation(selectedColumn, new SQLite3Select(auxiliaryQuery)); - SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(INOperation); + InOperation inOperation = new InOperation(selectedColumn, new SQLite3Select(auxiliaryQuery)); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(inOperation); originalQuery = this.genSelectExpression(selectedTable, specificCondition); originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); // folded query - // can not use IN VALUES here, because there is no affinity for the right operand of IN when right operand is a list + // can not use IN VALUES here, because there is no affinity for the right operand of IN when right operand + // is a list try { SQLite3Table t = this.createTemporaryTable(auxiliaryQuery, "intable"); SQLite3TableReference equivalentTable = new SQLite3TableReference(t); - INOperation = new InOperation(selectedColumn, equivalentTable); - specificCondition.updateInnerExpr(INOperation); + inOperation = new InOperation(selectedColumn, equivalentTable); + specificCondition.updateInnerExpr(inOperation); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } finally { dropTemporaryTable("intable"); } - } - // There is not `ANY` and `ALL` operator in SQLite3 - // Row Subquery - else { + } else { + // There is not `ANY` and `ALL` operator in SQLite3 + // Row Subquery // original query - SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); + SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.TEMP_TABLE_NAME); originalQuery = this.genSelectExpression(temporaryTable, null); SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); SQLite3WithClause withClause = new SQLite3WithClause(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); @@ -207,7 +208,7 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); // folded query - if (Randomly.getBoolean() && this.testCommonTableExpression()) { + if (Randomly.getBoolean() && this.enableCommonTableExpression()) { // there are too many false positives // common table expression // folded query: WITH table AS VALUES () @@ -215,25 +216,25 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro originalQuery.updateWithClauseRight(values); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } else if (Randomly.getBoolean() && this.testDerivedTable()) { + } else if (Randomly.getBoolean() && this.enableDerivedTable()) { // derived table // folded query: SELECT FROM () AS table originalQuery.setWithClause(null); SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); - originalQuery.replaceFromTable(this.tempTableName, alias); + originalQuery.replaceFromTable(this.TEMP_TABLE_NAME, alias); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); - } else if (this.testInsert()){ + } else if (this.enableInsert()) { // there are too many false positives // folded query: CREATE the table and INSERT INTO table subquery try { - this.createTemporaryTable(auxiliaryQuery, this.tempTableName); + this.createTemporaryTable(auxiliaryQuery, this.TEMP_TABLE_NAME); originalQuery.setWithClause(null); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } finally { - dropTemporaryTable(this.tempTableName); + dropTemporaryTable(this.TEMP_TABLE_NAME); } } else { throw new IgnoreMeException(); @@ -247,8 +248,10 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro } if (!compareResult(foldedResult, originalResult)) { reproducer = null; // TODO - state.getState().getLocalState().log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); - throw new AssertionError(auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); + state.getState().getLocalState() + .log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError( + auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); } } @@ -257,7 +260,7 @@ private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Express if (tempTable != null) { randomTables.addTable(tempTable); } - if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) { for (SQLite3Table t : this.tablesFromOuterContext) { randomTables.addTable(t); } @@ -270,7 +273,8 @@ private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Express } List columns = randomTables.getColumns(); - if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + if ((!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) + && this.joinsInExpr != null) { for (Join j : this.joinsInExpr) { SQLite3Table t = j.getTable(); columns.addAll(t.getColumns()); @@ -279,34 +283,36 @@ private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Express gen = new SQLite3ExpressionGenerator(state).setColumns(columns); List tables = randomTables.getTables(); List joinStatements = new ArrayList<>(); - if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (!useSubqueryAsFoldedExpr || useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr) { if (this.joinsInExpr != null) { joinStatements.addAll(this.joinsInExpr); this.joinsInExpr = null; } - } - else if (Randomly.getBoolean()) { - joinStatements = genJoinExpression(gen, tables, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); + } else if (Randomly.getBoolean()) { + joinStatements = genJoinExpression(gen, tables, + Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); } List tableRefs = SQLite3Common.getTableRefs(tables, s); SQLite3Select select = new SQLite3Select(); select.setFromList(tableRefs); - if (joinStatements.size() > 0) { + if (!joinStatements.isEmpty()) { select.setJoinClauses(joinStatements); } - + SQLite3Expression randomWhereCondition = gen.generateExpression(); SQLite3Expression whereCondition = null; if (specificCondition != null) { BinaryOperator operator = BinaryOperator.getRandomOperator(); - whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomWhereCondition, specificCondition, operator); + whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomWhereCondition, specificCondition, + operator); } else { whereCondition = randomWhereCondition; } select.setWhereClause(whereCondition); - + if (Randomly.getBoolean()) { - select.setOrderByClauses(genOrderBysExpression(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + select.setOrderByClauses(genOrderBysExpression(gen, + Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); } if (Randomly.getBoolean()) { @@ -314,7 +320,7 @@ else if (Randomly.getBoolean()) { List selectedAlias = new LinkedList<>(); for (int i = 0; i < selectedColumns.size(); ++i) { SQLite3ColumnName originalName = new SQLite3ColumnName(selectedColumns.get(i), null); - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(i)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + i), null); SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); selectedAlias.add(columnAlias); } @@ -322,14 +328,15 @@ else if (Randomly.getBoolean()) { } else { SQLite3ColumnName aggr = new SQLite3ColumnName(Randomly.fromList(columns), null); SQLite3Provider.mustKnowResult = true; - SQLite3Expression originalName = new SQLite3Aggregate(Arrays.asList(aggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + SQLite3Expression originalName = new SQLite3Aggregate(Arrays.asList(aggr), + SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c0"), null); SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); select.setFetchColumns(Arrays.asList(columnAlias)); if (Randomly.getBooleanWithRatherLowProbability()) { List groupByClause = genGroupByClause(columns, specificCondition); select.setGroupByClause(groupByClause); - if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + if (!groupByClause.isEmpty() && Randomly.getBooleanWithRatherLowProbability()) { select.setHavingClause(genHavingClause(columns, specificCondition)); } } @@ -349,13 +356,13 @@ private SQLite3Select genSimpleSelect() { if (Randomly.getBooleanWithRatherLowProbability()) { joinsInExpr = genJoinExpression(gen, tables, null, true); } else { - joinsInExpr = new ArrayList(); + joinsInExpr = new ArrayList<>(); } List tableRefs = SQLite3Common.getTableRefs(tables, s); SQLite3Select select = new SQLite3Select(); select.setFromList(tableRefs); - if (joinsInExpr != null && joinsInExpr.size() > 0) { + if (joinsInExpr != null && !joinsInExpr.isEmpty()) { select.setJoinClauses(joinsInExpr); } @@ -366,14 +373,14 @@ private SQLite3Select genSimpleSelect() { int columnIdx = 0; for (SQLite3Column c : randomTables.getColumns()) { SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); fetchColumns.add(columnAlias); columnIdx++; } // add the expression to fetch clause - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); SQLite3Alias columnAlias = new SQLite3Alias(whereCondition, aliasName); fetchColumns.add(columnAlias); @@ -388,7 +395,7 @@ private SQLite3Select genSimpleSelect() { } else { throw new AssertionError(e.getMessage()); } - } + } if (queryRes.get("c0").size() == 0) { throw new IgnoreMeException(); } @@ -398,7 +405,7 @@ private SQLite3Select genSimpleSelect() { selectResult.putAll(queryRes); // get the summary from results - List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + List summary = queryRes.remove("c" + columnIdx); List tempColumnList = new ArrayList<>(); @@ -443,13 +450,15 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { SQLite3Column newColumn = new SQLite3Column(c.getName(), c.getType(), false, null, false); newColumns.add(newColumn); } - SQLite3Table newTable = new SQLite3Table(t.getName() + "a", newColumns, null, true, false, false, false); + SQLite3Table newTable = new SQLite3Table(t.getName() + "a", newColumns, null, true, false, false, + false); for (SQLite3Column c : newColumns) { c.setTable(newTable); } innerQueryRandomTables.addTable(newTable); - - SQLite3Alias alias = new SQLite3Alias(new SQLite3TableReference(t), new SQLite3TableReference(newTable)); + + SQLite3Alias alias = new SQLite3Alias(new SQLite3TableReference(t), + new SQLite3TableReference(newTable)); innerQueryFromTables.add(alias); } } @@ -466,20 +475,21 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { innerQuery.setWhereClause(innerQueryWhereCondition); // use aggregate function in fetch column - SQLite3ColumnName innerQueryAggr = new SQLite3ColumnName(Randomly.fromList(innerQueryRandomTables.getColumns()), null); + SQLite3ColumnName innerQueryAggr = new SQLite3ColumnName(Randomly.fromList(innerQueryRandomTables.getColumns()), + null); SQLite3Provider.mustKnowResult = true; - SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), + SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); if (Randomly.getBooleanWithRatherLowProbability()) { List groupByClause = genGroupByClause(innerQueryColumns, null); innerQuery.setGroupByClause(groupByClause); - if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + if (!groupByClause.isEmpty() && Randomly.getBooleanWithRatherLowProbability()) { innerQuery.setHavingClause(genHavingClause(innerQueryColumns, null)); } } - this.foldedExpr = innerQuery; - + this.foldedExpr = innerQuery; // outer query SQLite3Select outerQuery = new SQLite3Select(); @@ -490,14 +500,14 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { int columnIdx = 0; for (SQLite3Column c : outerQueryRandomTables.getColumns()) { SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); outerQueryFetchColumns.add(columnAlias); columnIdx++; } // add the expression to fetch clause - SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + columnIdx), null); SQLite3Alias columnAlias = new SQLite3Alias(innerQuery, aliasName); outerQueryFetchColumns.add(columnAlias); @@ -514,7 +524,7 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { } else { throw new AssertionError(e.getMessage()); } - } + } if (queryRes.get("c0").size() == 0) { throw new IgnoreMeException(); } @@ -524,7 +534,7 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { selectResult.putAll(queryRes); // get the summary from results - List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + List summary = queryRes.remove("c" + columnIdx); List tempColumnList = new ArrayList<>(); @@ -550,7 +560,8 @@ private SQLite3Select genSelectWithCorrelatedSubquery() { return outerQuery; } - private List genJoinExpression(SQLite3ExpressionGenerator gen, List tables, SQLite3Expression specificCondition, boolean joinForExperssion) { + private List genJoinExpression(SQLite3ExpressionGenerator gen, List tables, + SQLite3Expression specificCondition, boolean joinForExperssion) { List joinStatements = new ArrayList<>(); if (!state.getDbmsSpecificOptions().testJoins) { return joinStatements; @@ -569,7 +580,8 @@ private List genJoinExpression(SQLite3ExpressionGenerator gen, List genJoinExpression(SQLite3ExpressionGenerator gen, List genOrderBysExpression(SQLite3ExpressionGenerator gen, SQLite3Expression specificCondition) { + private List genOrderBysExpression(SQLite3ExpressionGenerator gen, + SQLite3Expression specificCondition) { List expressions = new ArrayList<>(); for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - expressions.add(genOrderingTerm(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + expressions.add( + genOrderingTerm(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); } return expressions; } @@ -645,7 +659,8 @@ private SQLite3Expression genHavingClause(List columns, SQLite3Ex return expr; } - private Map> getQueryResult(String queryString, SQLite3GlobalState state) throws SQLException { + private Map> getQueryResult(String queryString, SQLite3GlobalState state) + throws SQLException { Map> result = new LinkedHashMap<>(); if (options.logEachSelect()) { logger.writeCurrentNoLineBreak(queryString); @@ -661,8 +676,8 @@ private Map> getQueryResult(String queryString, SQ Integer columnCount = metaData.getColumnCount(); Map idxNameMap = new HashMap<>(); for (int i = 1; i <= columnCount; i++) { - result.put("c" + String.valueOf(i-1), new ArrayList<>()); - idxNameMap.put(i, "c" + String.valueOf(i-1)); + result.put("c" + (i - 1), new ArrayList<>()); + idxNameMap.put(i, "c" + (i - 1)); } int resultRows = 0; @@ -673,26 +688,18 @@ private Map> getQueryResult(String queryString, SQ SQLite3Constant constant; if (rs.wasNull()) { constant = SQLite3Constant.createNullConstant(); - } - - else if (value instanceof Integer) { + } else if (value instanceof Integer) { constant = SQLite3Constant.createIntConstant(Long.valueOf((Integer) value)); } else if (value instanceof Short) { constant = SQLite3Constant.createIntConstant(Long.valueOf((Short) value)); } else if (value instanceof Long) { constant = SQLite3Constant.createIntConstant((Long) value); - } - - else if (value instanceof Double) { + } else if (value instanceof Double) { constant = SQLite3Constant.createRealConstant((double) value); } else if (value instanceof Float) { constant = SQLite3Constant.createRealConstant(((Float) value).doubleValue()); } else if (value instanceof BigDecimal) { constant = SQLite3Constant.createRealConstant(((BigDecimal) value).doubleValue()); - } - - else if (value instanceof Byte) { - constant = SQLite3Constant.createBinaryConstant((byte[]) value); } else if (value instanceof byte[]) { constant = SQLite3Constant.createBinaryConstant((byte[]) value); } else if (value instanceof Boolean) { @@ -746,7 +753,7 @@ private SQLite3Table genTemporaryTable(SQLite3Select select, String tableName) { List databaseColumns = new ArrayList<>(); for (int i = 0; i < columnNumber; ++i) { - String columnName = "c" + String.valueOf(i); + String columnName = "c" + i; SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); databaseColumns.add(column); } @@ -769,19 +776,19 @@ private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName String columnTypeName = ""; if (idxTypeMap.get(i) != null) { switch (idxTypeMap.get(i)) { - case INT: - case TEXT: - case REAL: - columnTypeName = idxTypeMap.get(i).name(); - break; - case BINARY: - columnTypeName = ""; - break; - default: - columnTypeName = ""; + case INT: + case TEXT: + case REAL: + columnTypeName = idxTypeMap.get(i).name(); + break; + case BINARY: + columnTypeName = ""; + break; + default: + columnTypeName = ""; } } - sb.append("c" + String.valueOf(i) + " " + columnTypeName); + sb.append("c" + i + " " + columnTypeName); if (i < columnNumber - 1) { sb.append(", "); } @@ -808,7 +815,7 @@ private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName } StringBuilder sb2 = new StringBuilder(); - sb2.append("INSERT INTO " + tableName + " "+ selectString); + sb2.append("INSERT INTO " + tableName + " " + selectString); String insertValueString = sb2.toString(); if (options.logEachSelect()) { logger.writeCurrent(insertValueString); @@ -831,7 +838,7 @@ private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName List databaseColumns = new ArrayList<>(); for (int i = 0; i < columnNumber; ++i) { - String columnName = "c" + String.valueOf(i); + String columnName = "c" + i; SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); databaseColumns.add(column); } @@ -869,20 +876,20 @@ private boolean compareResult(Map> r1, Map > entry: r1.entrySet()) { + for (Map.Entry> entry : r1.entrySet()) { String currentKey = entry.getKey(); if (!r2.containsKey(currentKey)) { return false; - } - List v1= entry.getValue(); - List v2= r2.get(currentKey); + } + List v1 = entry.getValue(); + List v2 = r2.get(currentKey); if (v1.size() != v2.size()) { return false; } List v1Value = new ArrayList<>(v1.stream().map(c -> c.toString()).collect(Collectors.toList())); List v2Value = new ArrayList<>(v2.stream().map(c -> c.toString()).collect(Collectors.toList())); Collections.sort(v1Value); - Collections.sort(v2Value); + Collections.sort(v2Value); if (!v1Value.equals(v2Value)) { return false; } @@ -893,7 +900,7 @@ private boolean compareResult(Map> r1, Map getColumnTypeFromSelect(SQLite3Select select) { List fetchColumns = select.getFetchColumns(); List newFetchColumns = new ArrayList<>(); - for(SQLite3Expression column : fetchColumns) { + for (SQLite3Expression column : fetchColumns) { newFetchColumns.add(column); SQLite3Alias columnAlias = (SQLite3Alias) column; SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOriginalExpression()); @@ -910,14 +917,14 @@ private Map getColumnTypeFromSelect(SQLite3Select sele } else { throw new AssertionError(e.getMessage()); } - } + } if (typeResult == null) { throw new IgnoreMeException(); } Map idxTypeMap = new HashMap<>(); for (int i = 0; i * 2 < typeResult.size(); ++i) { - String columnName = "c" + String.valueOf(i * 2 + 1); + String columnName = "c" + (i * 2 + 1); SQLite3Expression t = typeResult.get(columnName).get(0); SQLite3TextConstant tString = (SQLite3TextConstant) t; String typeName = tString.asString(); @@ -946,16 +953,19 @@ public boolean useCorrelatedSubquery() { return Randomly.getBoolean(); } - public boolean testCommonTableExpression() { + public boolean enableCommonTableExpression() { return false; } - public boolean testDerivedTable() { + + public boolean enableDerivedTable() { return true; } - public boolean testInsert() { + + public boolean enableInsert() { return false; } - public boolean testInOperator() { + + public boolean enableInOperator() { return false; } @@ -968,4 +978,4 @@ public String getLastQueryString() { public Reproducer getLastReproducer() { return reproducer; } -} \ No newline at end of file +} diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index 556976056..b7145b98c 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -21,20 +21,21 @@ public static SQLite3DataType getTypeFromName(String name) { } } + @Override public String toString() { - switch(this) { - case BINARY: - return "BLOB"; - case INT: - return "INT"; - case TEXT: - return "TEXT"; - case REAL: - return "REAL"; - case NULL: - return "NULL"; - default: - throw new IgnoreMeException(); + switch (this) { + case BINARY: + return "BLOB"; + case INT: + return "INT"; + case TEXT: + return "TEXT"; + case REAL: + return "REAL"; + case NULL: + return "NULL"; + default: + throw new IgnoreMeException(); } } } diff --git a/test/sqlancer/TestCommonSchema.java b/test/sqlancer/TestCommonSchema.java index c77892124..f3a2ee2ba 100644 --- a/test/sqlancer/TestCommonSchema.java +++ b/test/sqlancer/TestCommonSchema.java @@ -63,7 +63,7 @@ private TestSchema createTestSchema(TestTable... tables) { } private TestTables createTestTables(TestTable... tables) { - return new TestTables(new ArrayList(Arrays.asList(tables))); + return new TestTables(new ArrayList(Arrays.asList(tables))); } @Test diff --git a/test/sqlancer/dbms/TestSQLiteCODDTest.java b/test/sqlancer/dbms/TestSQLiteCODDTest.java index 6697c09c5..c1948a9b8 100644 --- a/test/sqlancer/dbms/TestSQLiteCODDTest.java +++ b/test/sqlancer/dbms/TestSQLiteCODDTest.java @@ -10,9 +10,7 @@ public class TestSQLiteCODDTest { @Test public void testSqliteCODDTest() { - assertEquals(0, - Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, - "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", - "CODDTest" })); + assertEquals(0, Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--num-threads", "1", "--num-queries", TestConfig.NUM_QUERIES, "sqlite3", "--oracle", "CODDTest" })); } } From dcdf7d641903051a65b8153f01c7c7cf44a0254d Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Wed, 23 Apr 2025 19:11:09 +0800 Subject: [PATCH 669/856] remove three errors, and fix a warning about the using of static variable --- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index c262243a9..532709676 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -81,13 +81,13 @@ public SQLite3CODDTestOracle(SQLite3GlobalState globalState) { SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addMatchQueryErrors(errors); SQLite3Errors.addQueryErrors(errors); - errors.add("misuse of aggregate"); - errors.add("misuse of window function"); - errors.add("second argument to nth_value must be a positive integer"); - errors.add("no such table"); - errors.add("no query solution"); - errors.add("unable to use function MATCH in the requested context"); - errors.add("[SQLITE_ERROR] SQL error or missing database (unrecognized token:"); + // errors.add("misuse of aggregate"); + // errors.add("misuse of window function"); + // errors.add("second argument to nth_value must be a positive integer"); + // errors.add("no such table"); + // errors.add("no query solution"); + // errors.add("unable to use function MATCH in the requested context"); + // errors.add("[SQLITE_ERROR] SQL error or missing database (unrecognized token:"); } @Override @@ -200,7 +200,7 @@ && enableInOperator()) { // There is not `ANY` and `ALL` operator in SQLite3 // Row Subquery // original query - SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.TEMP_TABLE_NAME); + SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, SQLite3CODDTestOracle.TEMP_TABLE_NAME); originalQuery = this.genSelectExpression(temporaryTable, null); SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); SQLite3WithClause withClause = new SQLite3WithClause(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); @@ -222,19 +222,19 @@ && enableInOperator()) { originalQuery.setWithClause(null); SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); - originalQuery.replaceFromTable(this.TEMP_TABLE_NAME, alias); + originalQuery.replaceFromTable(SQLite3CODDTestOracle.TEMP_TABLE_NAME, alias); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } else if (this.enableInsert()) { // there are too many false positives // folded query: CREATE the table and INSERT INTO table subquery try { - this.createTemporaryTable(auxiliaryQuery, this.TEMP_TABLE_NAME); + this.createTemporaryTable(auxiliaryQuery, SQLite3CODDTestOracle.TEMP_TABLE_NAME); originalQuery.setWithClause(null); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } finally { - dropTemporaryTable(this.TEMP_TABLE_NAME); + dropTemporaryTable(SQLite3CODDTestOracle.TEMP_TABLE_NAME); } } else { throw new IgnoreMeException(); From 597b43ab3f94fd36b71e19f256edc5850a34587d Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Wed, 23 Apr 2025 23:09:46 +0800 Subject: [PATCH 670/856] remove the redundant code --- .../sqlite3/schema/SQLite3DataType.java | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index b7145b98c..8a343fed2 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -20,22 +20,4 @@ public static SQLite3DataType getTypeFromName(String name) { throw new IgnoreMeException(); } } - - @Override - public String toString() { - switch (this) { - case BINARY: - return "BLOB"; - case INT: - return "INT"; - case TEXT: - return "TEXT"; - case REAL: - return "REAL"; - case NULL: - return "NULL"; - default: - throw new IgnoreMeException(); - } - } } From b2d028434f63b10ab9c82757bd6c989a41a484d8 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 24 Apr 2025 00:16:45 +0800 Subject: [PATCH 671/856] format hive code --- src/sqlancer/hive/HiveOptions.java | 4 +- src/sqlancer/hive/HiveProvider.java | 12 +- src/sqlancer/hive/HiveToStringVisitor.java | 40 +++--- .../hive/ast/HiveBetweenOperation.java | 8 +- .../hive/ast/HiveBinaryOperation.java | 5 +- src/sqlancer/hive/ast/HiveCaseOperation.java | 2 +- src/sqlancer/hive/ast/HiveCastOperation.java | 8 +- src/sqlancer/hive/ast/HiveFunction.java | 2 +- src/sqlancer/hive/ast/HiveInOperation.java | 2 +- src/sqlancer/hive/ast/HiveJoin.java | 2 +- src/sqlancer/hive/ast/HiveSelect.java | 2 +- src/sqlancer/hive/ast/HiveTableReference.java | 2 +- .../hive/ast/HiveUnaryPostfixOperation.java | 3 +- .../hive/ast/HiveUnaryPrefixOperation.java | 5 +- .../hive/gen/HiveExpressionGenerator.java | 129 +++++++----------- src/sqlancer/hive/gen/HiveTableGenerator.java | 52 +++---- test/sqlancer/dbms/TestHiveTLP.java | 8 +- 17 files changed, 127 insertions(+), 159 deletions(-) diff --git a/src/sqlancer/hive/HiveOptions.java b/src/sqlancer/hive/HiveOptions.java index a8c049756..5d2ea47e6 100644 --- a/src/sqlancer/hive/HiveOptions.java +++ b/src/sqlancer/hive/HiveOptions.java @@ -28,8 +28,8 @@ public enum HiveOracleFactory implements OracleFactory { @Override public TestOracle create(HiveGlobalState globalState) throws SQLException { HiveExpressionGenerator gen = new HiveExpressionGenerator(globalState); - ExpectedErrors expectedErrors = ExpectedErrors.newErrors() - .with(HiveErrors.getExpressionErrors()).build(); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(HiveErrors.getExpressionErrors()) + .build(); return new TLPWhereOracle<>(globalState, gen, expectedErrors); } diff --git a/src/sqlancer/hive/HiveProvider.java b/src/sqlancer/hive/HiveProvider.java index cf1df5410..723e3fd6f 100644 --- a/src/sqlancer/hive/HiveProvider.java +++ b/src/sqlancer/hive/HiveProvider.java @@ -61,15 +61,14 @@ public void generateDatabase(HiveGlobalState globalState) throws Exception { String tableName = globalState.getSchema().getFreeTableName(); SQLQueryAdapter qt = HiveTableGenerator.generate(globalState, tableName); success = globalState.executeStatement(qt); - } while(!success); + } while (!success); } if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); // TODO } - StatementExecutor se = new StatementExecutor( - globalState, Action.values(), - HiveProvider::mapActions, (q) -> { + StatementExecutor se = new StatementExecutor(globalState, + Action.values(), HiveProvider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } @@ -107,9 +106,8 @@ public SQLConnection createDatabase(HiveGlobalState globalState) throws SQLExcep s.execute("USE " + databaseName); } con.close(); - con = DriverManager.getConnection( - String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName, - username, password)); + con = DriverManager + .getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName, username, password)); return new SQLConnection(con); } diff --git a/src/sqlancer/hive/HiveToStringVisitor.java b/src/sqlancer/hive/HiveToStringVisitor.java index 3815561d8..d23818916 100644 --- a/src/sqlancer/hive/HiveToStringVisitor.java +++ b/src/sqlancer/hive/HiveToStringVisitor.java @@ -71,26 +71,26 @@ private void visit(HiveSelect select) { private void visit(HiveJoin join) { switch (join.getJoinType()) { - case INNER: - sb.append(" INNER JOIN "); - break; - case LEFT_OUTER: - sb.append(" LEFT JOIN "); - break; - case RIGHT_OUTER: - sb.append(" RIGHT JOIN "); - break; - case FULL_OUTER: - sb.append(" FULL JOIN "); - break; - case LEFT_SEMI: - sb.append(" LEFT SEMI JOIN "); - break; - case CROSS: - sb.append(" CROSS JOIN "); - break; - default: - throw new UnsupportedOperationException(); + case INNER: + sb.append(" INNER JOIN "); + break; + case LEFT_OUTER: + sb.append(" LEFT JOIN "); + break; + case RIGHT_OUTER: + sb.append(" RIGHT JOIN "); + break; + case FULL_OUTER: + sb.append(" FULL JOIN "); + break; + case LEFT_SEMI: + sb.append(" LEFT SEMI JOIN "); + break; + case CROSS: + sb.append(" CROSS JOIN "); + break; + default: + throw new UnsupportedOperationException(); } visit((TableReferenceNode) join.getRightTable()); if (join.getOnClause() != null) { diff --git a/src/sqlancer/hive/ast/HiveBetweenOperation.java b/src/sqlancer/hive/ast/HiveBetweenOperation.java index a65ef70d2..26ec1d940 100644 --- a/src/sqlancer/hive/ast/HiveBetweenOperation.java +++ b/src/sqlancer/hive/ast/HiveBetweenOperation.java @@ -2,11 +2,9 @@ import sqlancer.common.ast.newast.NewBetweenOperatorNode; -public class HiveBetweenOperation extends NewBetweenOperatorNode - implements HiveExpression { - - public HiveBetweenOperation(HiveExpression left, HiveExpression middle, HiveExpression right, - boolean isTrue) { +public class HiveBetweenOperation extends NewBetweenOperatorNode implements HiveExpression { + + public HiveBetweenOperation(HiveExpression left, HiveExpression middle, HiveExpression right, boolean isTrue) { super(left, middle, right, isTrue); } } diff --git a/src/sqlancer/hive/ast/HiveBinaryOperation.java b/src/sqlancer/hive/ast/HiveBinaryOperation.java index 61d9cf104..f74d117d4 100644 --- a/src/sqlancer/hive/ast/HiveBinaryOperation.java +++ b/src/sqlancer/hive/ast/HiveBinaryOperation.java @@ -3,9 +3,8 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewBinaryOperatorNode; -public class HiveBinaryOperation extends NewBinaryOperatorNode - implements HiveExpression { - +public class HiveBinaryOperation extends NewBinaryOperatorNode implements HiveExpression { + public HiveBinaryOperation(HiveExpression left, HiveExpression right, Operator op) { super(left, right, op); } diff --git a/src/sqlancer/hive/ast/HiveCaseOperation.java b/src/sqlancer/hive/ast/HiveCaseOperation.java index 2a52a5c45..666070667 100644 --- a/src/sqlancer/hive/ast/HiveCaseOperation.java +++ b/src/sqlancer/hive/ast/HiveCaseOperation.java @@ -5,7 +5,7 @@ import sqlancer.common.ast.newast.NewCaseOperatorNode; public class HiveCaseOperation extends NewCaseOperatorNode implements HiveExpression { - + public HiveCaseOperation(HiveExpression switchCondition, List conditions, List expressions, HiveExpression elseExpr) { super(switchCondition, conditions, expressions, elseExpr); diff --git a/src/sqlancer/hive/ast/HiveCastOperation.java b/src/sqlancer/hive/ast/HiveCastOperation.java index ae2fbe435..2d76ab4f2 100644 --- a/src/sqlancer/hive/ast/HiveCastOperation.java +++ b/src/sqlancer/hive/ast/HiveCastOperation.java @@ -3,10 +3,10 @@ import sqlancer.hive.HiveSchema.HiveDataType; public class HiveCastOperation implements HiveExpression { - + private final HiveExpression expression; private final HiveDataType type; - + public HiveCastOperation(HiveExpression expression, HiveDataType type) { if (expression == null) { throw new AssertionError(); @@ -14,11 +14,11 @@ public HiveCastOperation(HiveExpression expression, HiveDataType type) { this.expression = expression; this.type = type; } - + public HiveExpression getExpression() { return expression; } - + public HiveDataType getType() { return type; } diff --git a/src/sqlancer/hive/ast/HiveFunction.java b/src/sqlancer/hive/ast/HiveFunction.java index 507015259..b3a34ae25 100644 --- a/src/sqlancer/hive/ast/HiveFunction.java +++ b/src/sqlancer/hive/ast/HiveFunction.java @@ -9,5 +9,5 @@ public class HiveFunction extends NewFunctionNode implemen public HiveFunction(List args, F func) { super(args, func); } - + } diff --git a/src/sqlancer/hive/ast/HiveInOperation.java b/src/sqlancer/hive/ast/HiveInOperation.java index ccffde58c..601bf5e19 100644 --- a/src/sqlancer/hive/ast/HiveInOperation.java +++ b/src/sqlancer/hive/ast/HiveInOperation.java @@ -5,7 +5,7 @@ import sqlancer.common.ast.newast.NewInOperatorNode; public class HiveInOperation extends NewInOperatorNode implements HiveExpression { - + public HiveInOperation(HiveExpression left, List right, boolean isNegated) { super(left, right, isNegated); } diff --git a/src/sqlancer/hive/ast/HiveJoin.java b/src/sqlancer/hive/ast/HiveJoin.java index fc6b7a200..932ed9afa 100644 --- a/src/sqlancer/hive/ast/HiveJoin.java +++ b/src/sqlancer/hive/ast/HiveJoin.java @@ -17,7 +17,7 @@ public enum JoinType { INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, CROSS; } - public HiveJoin(HiveTableReference leftTable, HiveTableReference rightTable, JoinType joinType, + public HiveJoin(HiveTableReference leftTable, HiveTableReference rightTable, JoinType joinType, HiveExpression onClause) { this.leftTable = leftTable; this.rightTable = rightTable; diff --git a/src/sqlancer/hive/ast/HiveSelect.java b/src/sqlancer/hive/ast/HiveSelect.java index 4853ef892..b099ade02 100644 --- a/src/sqlancer/hive/ast/HiveSelect.java +++ b/src/sqlancer/hive/ast/HiveSelect.java @@ -8,7 +8,7 @@ import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveTable; -public class HiveSelect extends SelectBase +public class HiveSelect extends SelectBase implements Select, HiveExpression { private boolean isDistinct; diff --git a/src/sqlancer/hive/ast/HiveTableReference.java b/src/sqlancer/hive/ast/HiveTableReference.java index a0955f86c..a1b586825 100644 --- a/src/sqlancer/hive/ast/HiveTableReference.java +++ b/src/sqlancer/hive/ast/HiveTableReference.java @@ -3,7 +3,7 @@ import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.hive.HiveSchema; -public class HiveTableReference extends TableReferenceNode +public class HiveTableReference extends TableReferenceNode implements HiveExpression { public HiveTableReference(HiveSchema.HiveTable table) { diff --git a/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java index c32a40847..0461c5c73 100644 --- a/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java +++ b/src/sqlancer/hive/ast/HiveUnaryPostfixOperation.java @@ -3,8 +3,7 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; -public class HiveUnaryPostfixOperation extends NewUnaryPostfixOperatorNode - implements HiveExpression { +public class HiveUnaryPostfixOperation extends NewUnaryPostfixOperatorNode implements HiveExpression { public HiveUnaryPostfixOperation(HiveExpression expr, Operator op) { super(expr, op); diff --git a/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java index 9312c4c26..9fe286f82 100644 --- a/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java +++ b/src/sqlancer/hive/ast/HiveUnaryPrefixOperation.java @@ -3,11 +3,10 @@ import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; -public class HiveUnaryPrefixOperation extends NewUnaryPrefixOperatorNode - implements HiveExpression { +public class HiveUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements HiveExpression { public HiveUnaryPrefixOperation(HiveExpression expr, Operator op) { super(expr, op); } - + } diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index 3e35544c5..bd9134de6 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -29,7 +29,7 @@ import java.util.List; import java.util.stream.Collectors; -public class HiveExpressionGenerator extends UntypedExpressionGenerator +public class HiveExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator { private final HiveGlobalState globalState; @@ -37,8 +37,8 @@ public class HiveExpressionGenerator extends UntypedExpressionGenerator= globalState.getOptions().getMaxExpressionDepth() + if (depth >= globalState.getOptions().getMaxExpressionDepth() || Randomly.getBooleanWithRatherLowProbability()) { return generateLeafNode(); } if (allowAggregates && Randomly.getBooleanWithRatherLowProbability()) { - allowAggregates= false; // aggregate function calls cannot be nested + allowAggregates = false; // aggregate function calls cannot be nested HiveAggregateFunction aggregate = HiveAggregateFunction.getRandom(); return new HiveFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); } @@ -77,46 +77,36 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr Expression expr = Randomly.fromList(possibleOptions); switch (expr) { - case UNARY_PREFIX: - return new HiveUnaryPrefixOperation(generateExpression(depth + 1), - HiveUnaryPrefixOperator.getRandom()); - case UNARY_POSTFIX: - return new HiveUnaryPostfixOperation(generateExpression(depth + 1), - HiveUnaryPostfixOperator.getRandom()); - case BINARY_COMPARISON: - Operator op = HiveBinaryComparisonOperator.getRandom(); - return new HiveBinaryOperation(generateExpression(depth + 1), - generateExpression(depth + 1), op); - case BINARY_LOGICAL: - op = HiveExpressionGenerator.HiveBinaryLogicalOperator.getRandom(); - return new HiveBinaryOperation(generateExpression(depth + 1), - generateExpression(depth + 1), op); - case BINARY_ARITHMETIC: - return new HiveBinaryOperation(generateExpression(depth + 1), - generateExpression(depth + 1), - HiveExpressionGenerator.HiveBinaryArithmeticOperator.getRandom()); - case CAST: - return new HiveCastOperation(generateExpression(depth + 1), HiveDataType.getRandomType()); - case FUNC: - HiveFunc func = HiveFunc.getRandom(); - return new HiveFunction<>(generateExpressions(func.getNrArgs()), func); - case BETWEEN: - return new HiveBetweenOperation(generateExpression(depth + 1), - generateExpression(depth + 1), - generateExpression(depth + 1), - Randomly.getBoolean()); - case IN: - return new HiveInOperation(generateExpression(depth + 1), - generateExpressions(Randomly.smallNumber() + 1, depth + 1), - Randomly.getBoolean()); - case CASE: - int nr = Randomly.smallNumber() + 1; - return new HiveCaseOperation(generateExpression(depth + 1), - generateExpressions(nr, depth + 1), - generateExpressions(nr, depth + 1), - generateExpression(depth + 1)); - default: - throw new AssertionError(expr); + case UNARY_PREFIX: + return new HiveUnaryPrefixOperation(generateExpression(depth + 1), HiveUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new HiveUnaryPostfixOperation(generateExpression(depth + 1), HiveUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = HiveBinaryComparisonOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = HiveExpressionGenerator.HiveBinaryLogicalOperator.getRandom(); + return new HiveBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new HiveBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + HiveExpressionGenerator.HiveBinaryArithmeticOperator.getRandom()); + case CAST: + return new HiveCastOperation(generateExpression(depth + 1), HiveDataType.getRandomType()); + case FUNC: + HiveFunc func = HiveFunc.getRandom(); + return new HiveFunction<>(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new HiveBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new HiveInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new HiveCaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(expr); } } @@ -128,16 +118,16 @@ public HiveExpression generateConstant() { HiveDataType[] values = HiveDataType.values(); HiveDataType constantType = Randomly.fromOptions(values); switch (constantType) { - case STRING: - return HiveConstant.createStringConstant(globalState.getRandomly().getString()); - case INT: - return HiveConstant.createIntConstant(globalState.getRandomly().getInteger()); - case DOUBLE: - return HiveConstant.createDoubleConstant(globalState.getRandomly().getDouble()); - case BOOLEAN: - return HiveConstant.createBooleanConstant(Randomly.getBoolean()); - default: - throw new AssertionError(constantType); + case STRING: + return HiveConstant.createStringConstant(globalState.getRandomly().getString()); + case INT: + return HiveConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DOUBLE: + return HiveConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BOOLEAN: + return HiveConstant.createBooleanConstant(Randomly.getBoolean()); + default: + throw new AssertionError(constantType); } } @@ -160,7 +150,6 @@ public List generateOrderBys() { return newExpr; } - @Override public HiveExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); @@ -189,8 +178,7 @@ public List generateFetchColumns(boolean allowAggregates) { if (Randomly.getBoolean()) { return List.of(new HiveColumnReference(new HiveColumn("*", null, null))); } - return Randomly.nonEmptySubset(columns).stream() - .map(c -> new HiveColumnReference(c)) + return Randomly.nonEmptySubset(columns).stream().map(c -> new HiveColumnReference(c)) .collect(Collectors.toList()); } @@ -243,9 +231,8 @@ public String getTextRepresentation() { public enum HiveBinaryComparisonOperator implements Operator { - EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), - SMALLER_EQUALS("<="), NOT_EQUALS("!="), LIKE("LIKE"), - NOT_LIKE("NOT LIKE"), REGEXP("RLIKE"); + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), REGEXP("RLIKE"); private String textRepr; @@ -305,18 +292,8 @@ public String getTextRepresentation() { } public enum HiveAggregateFunction { - COUNT(1), - SUM(1), - AVG(1), - MIN(1), - MAX(1), - VARIANCE(1), - VAR_SAMP(1), - STDDEV_POP(1), - STDDEV_SAMP(1), - COVAR_POP(2), - COVAR_SAMP(2), - CORR(2); + COUNT(1), SUM(1), AVG(1), MIN(1), MAX(1), VARIANCE(1), VAR_SAMP(1), STDDEV_POP(1), STDDEV_SAMP(1), COVAR_POP(2), + COVAR_SAMP(2), CORR(2); private int nrArgs; @@ -335,17 +312,15 @@ public int getNrArgs() { // TODO: test all Hive default functions... public enum HiveFunc { - + // mathematical functions - ROUND(2), - FLOOR(1); + ROUND(2), FLOOR(1); // collection functions // date functions // string functions - private int nrArgs; private boolean isVariadic; diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java index 43192a5be..6f57818fe 100644 --- a/src/sqlancer/hive/gen/HiveTableGenerator.java +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -82,8 +82,9 @@ private void appendColumn(int columnId, ExpectedErrors errors) { } private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { - /* column_constraint_specification: - : [ PRIMARY KEY|UNIQUE|NOT NULL|DEFAULT [default_value]|CHECK [check_expression] ENABLE|DISABLE NOVALIDATE RELY/NORELY ] + /* + * column_constraint_specification: : [ PRIMARY KEY|UNIQUE|NOT NULL|DEFAULT [default_value]|CHECK + * [check_expression] ENABLE|DISABLE NOVALIDATE RELY/NORELY ] */ if (Randomly.getBoolean()) { // no column constraint @@ -92,31 +93,30 @@ private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { ColumnConstraints constraint = Randomly.fromOptions(ColumnConstraints.values()); switch (constraint) { - case PRIMARY_KEY_DISABLE: - if (allowPrimaryKey && !setPrimaryKey) { - sb.append(" PRIMARY KEY DISABLE"); - setPrimaryKey = true; - } - break; - case UNIQUE_DISABLE: - sb.append(" UNIQUE DISABLE"); - break; - case NOT_NULL: - sb.append(" NOT NULL"); - break; - case DEFAULT: - sb.append(" DEFAULT ("); - sb.append(HiveToStringVisitor.asString(gen.generateConstant())); - sb.append(")"); - case CHECK: - sb.append(" CHECK ("); - sb.append(HiveToStringVisitor.asString(gen.generateExpression())); - sb.append(")"); - break; - default: - throw new AssertionError(constraint); + case PRIMARY_KEY_DISABLE: + if (allowPrimaryKey && !setPrimaryKey) { + sb.append(" PRIMARY KEY DISABLE"); + setPrimaryKey = true; + } + break; + case UNIQUE_DISABLE: + sb.append(" UNIQUE DISABLE"); + break; + case NOT_NULL: + sb.append(" NOT NULL"); + break; + case DEFAULT: + sb.append(" DEFAULT ("); + sb.append(HiveToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + case CHECK: + sb.append(" CHECK ("); + sb.append(HiveToStringVisitor.asString(gen.generateExpression())); + sb.append(")"); + break; + default: + throw new AssertionError(constraint); } } - } diff --git a/test/sqlancer/dbms/TestHiveTLP.java b/test/sqlancer/dbms/TestHiveTLP.java index 05a4bc20d..5f8692f3f 100644 --- a/test/sqlancer/dbms/TestHiveTLP.java +++ b/test/sqlancer/dbms/TestHiveTLP.java @@ -8,13 +8,13 @@ import sqlancer.Main; public class TestHiveTLP { - + @Test public void testHiveTLPWhere() { assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.HIVE_ENV)); assertEquals(0, - Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", - "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", - "--num-queries", TestConfig.NUM_QUERIES, "hive", "--oracle", "TLPWhere" })); + Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", + "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "hive", "--oracle", "TLPWhere" })); } } \ No newline at end of file From f52353050457570da5f5eea02bb11948b350c190 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 24 Apr 2025 00:18:44 +0800 Subject: [PATCH 672/856] add test of CODDTest for SQLite3 in github action --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8ea1059a6..888c04d06 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -569,6 +569,7 @@ jobs: mvn -Dtest=TestSQLitePQS test mvn -Dtest=TestSQLiteTLP test mvn -Dtest=TestSQLiteNoREC test + mvn -Dtest=TestSQLiteCODDTest test sqlite-qpg: name: QPG Tests (SQLite) From a9f7dce592ab056f8cf4a28a1da79a3ead589958 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 24 Apr 2025 09:30:09 +0800 Subject: [PATCH 673/856] format hive for mvn verify --- pom.xml | 2 +- src/sqlancer/hive/HiveErrors.java | 2 +- src/sqlancer/hive/HiveOptions.java | 12 ++++++------ src/sqlancer/hive/HiveProvider.java | 18 +++++++++--------- src/sqlancer/hive/HiveSchema.java | 16 ++++++++-------- src/sqlancer/hive/HiveToStringVisitor.java | 2 +- src/sqlancer/hive/ast/HiveSelect.java | 2 +- src/sqlancer/hive/ast/HiveTableReference.java | 2 +- .../hive/gen/HiveExpressionGenerator.java | 18 ++++++++++-------- src/sqlancer/hive/gen/HiveTableGenerator.java | 18 +++++++++--------- 10 files changed, 47 insertions(+), 45 deletions(-) diff --git a/pom.xml b/pom.xml index bf30df4ce..83068f498 100644 --- a/pom.xml +++ b/pom.xml @@ -378,7 +378,7 @@ flight-sql-jdbc-driver 16.1.0 - + org.apache.hive hive-jdbc 4.0.1 diff --git a/src/sqlancer/hive/HiveErrors.java b/src/sqlancer/hive/HiveErrors.java index b061c1071..81b0be668 100644 --- a/src/sqlancer/hive/HiveErrors.java +++ b/src/sqlancer/hive/HiveErrors.java @@ -5,7 +5,7 @@ import sqlancer.common.query.ExpectedErrors; -public class HiveErrors { +public final class HiveErrors { private HiveErrors() { } diff --git a/src/sqlancer/hive/HiveOptions.java b/src/sqlancer/hive/HiveOptions.java index 5d2ea47e6..ea55ac676 100644 --- a/src/sqlancer/hive/HiveOptions.java +++ b/src/sqlancer/hive/HiveOptions.java @@ -1,18 +1,18 @@ package sqlancer.hive; +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + import com.beust.jcommander.Parameter; import com.beust.jcommander.Parameters; +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; -import sqlancer.DBMSSpecificOptions; import sqlancer.hive.gen.HiveExpressionGenerator; -import sqlancer.OracleFactory; - -import java.sql.SQLException; -import java.util.Arrays; -import java.util.List; @Parameters(separators = "=", commandDescription = "Hive (default port: " + HiveOptions.DEFAULT_PORT + ", default host: " + HiveOptions.DEFAULT_HOST + ")") diff --git a/src/sqlancer/hive/HiveProvider.java b/src/sqlancer/hive/HiveProvider.java index 723e3fd6f..12798df93 100644 --- a/src/sqlancer/hive/HiveProvider.java +++ b/src/sqlancer/hive/HiveProvider.java @@ -1,5 +1,12 @@ package sqlancer.hive; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + import sqlancer.AbstractAction; import sqlancer.DatabaseProvider; import sqlancer.IgnoreMeException; @@ -13,13 +20,6 @@ import sqlancer.hive.gen.HiveInsertGenerator; import sqlancer.hive.gen.HiveTableGenerator; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.sql.Statement; - -import com.google.auto.service.AutoService; - @AutoService(DatabaseProvider.class) public class HiveProvider extends SQLProviderAdapter { @@ -67,8 +67,8 @@ public void generateDatabase(HiveGlobalState globalState) throws Exception { throw new IgnoreMeException(); // TODO } - StatementExecutor se = new StatementExecutor(globalState, - Action.values(), HiveProvider::mapActions, (q) -> { + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + HiveProvider::mapActions, (q) -> { if (globalState.getSchema().getDatabaseTables().isEmpty()) { throw new IgnoreMeException(); } diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java index fcc17c77c..822eea163 100644 --- a/src/sqlancer/hive/HiveSchema.java +++ b/src/sqlancer/hive/HiveSchema.java @@ -1,5 +1,13 @@ package sqlancer.hive; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + import sqlancer.Randomly; import sqlancer.SQLConnection; import sqlancer.common.schema.AbstractRelationalTable; @@ -9,14 +17,6 @@ import sqlancer.common.schema.TableIndex; import sqlancer.hive.HiveSchema.HiveTable; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - public class HiveSchema extends AbstractSchema { public enum HiveDataType { diff --git a/src/sqlancer/hive/HiveToStringVisitor.java b/src/sqlancer/hive/HiveToStringVisitor.java index d23818916..bdcd31eaf 100644 --- a/src/sqlancer/hive/HiveToStringVisitor.java +++ b/src/sqlancer/hive/HiveToStringVisitor.java @@ -2,11 +2,11 @@ import sqlancer.common.ast.newast.NewToStringVisitor; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.hive.ast.HiveCastOperation; import sqlancer.hive.ast.HiveConstant; import sqlancer.hive.ast.HiveExpression; import sqlancer.hive.ast.HiveJoin; import sqlancer.hive.ast.HiveSelect; -import sqlancer.hive.ast.HiveCastOperation; public class HiveToStringVisitor extends NewToStringVisitor { diff --git a/src/sqlancer/hive/ast/HiveSelect.java b/src/sqlancer/hive/ast/HiveSelect.java index b099ade02..8a0eddc06 100644 --- a/src/sqlancer/hive/ast/HiveSelect.java +++ b/src/sqlancer/hive/ast/HiveSelect.java @@ -4,9 +4,9 @@ import sqlancer.common.ast.SelectBase; import sqlancer.common.ast.newast.Select; -import sqlancer.hive.HiveToStringVisitor; import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveTable; +import sqlancer.hive.HiveToStringVisitor; public class HiveSelect extends SelectBase implements Select, HiveExpression { diff --git a/src/sqlancer/hive/ast/HiveTableReference.java b/src/sqlancer/hive/ast/HiveTableReference.java index a1b586825..7d23b4895 100644 --- a/src/sqlancer/hive/ast/HiveTableReference.java +++ b/src/sqlancer/hive/ast/HiveTableReference.java @@ -10,4 +10,4 @@ public HiveTableReference(HiveSchema.HiveTable table) { super(table); } -} \ No newline at end of file +} diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index bd9134de6..9f3de2514 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -1,12 +1,20 @@ package sqlancer.hive.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; import sqlancer.hive.HiveGlobalState; -import sqlancer.hive.HiveSchema.*; +import sqlancer.hive.HiveSchema.HiveColumn; +import sqlancer.hive.HiveSchema.HiveDataType; +import sqlancer.hive.HiveSchema.HiveTable; import sqlancer.hive.ast.HiveBetweenOperation; import sqlancer.hive.ast.HiveBinaryOperation; import sqlancer.hive.ast.HiveCaseOperation; @@ -20,14 +28,8 @@ import sqlancer.hive.ast.HiveOrderingTerm; import sqlancer.hive.ast.HiveSelect; import sqlancer.hive.ast.HiveTableReference; -import sqlancer.hive.ast.HiveUnaryPrefixOperation; import sqlancer.hive.ast.HiveUnaryPostfixOperation; -import sqlancer.common.ast.BinaryOperatorNode.Operator; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; +import sqlancer.hive.ast.HiveUnaryPrefixOperation; public class HiveExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator { diff --git a/src/sqlancer/hive/gen/HiveTableGenerator.java b/src/sqlancer/hive/gen/HiveTableGenerator.java index 6f57818fe..c1c4db2bd 100644 --- a/src/sqlancer/hive/gen/HiveTableGenerator.java +++ b/src/sqlancer/hive/gen/HiveTableGenerator.java @@ -1,5 +1,8 @@ package sqlancer.hive.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.DBMSCommon; import sqlancer.common.query.ExpectedErrors; @@ -7,13 +10,10 @@ import sqlancer.hive.HiveErrors; import sqlancer.hive.HiveGlobalState; import sqlancer.hive.HiveSchema; -import sqlancer.hive.HiveToStringVisitor; import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveDataType; import sqlancer.hive.HiveSchema.HiveTable; - -import java.util.ArrayList; -import java.util.List; +import sqlancer.hive.HiveToStringVisitor; public class HiveTableGenerator { @@ -31,7 +31,7 @@ private enum ColumnConstraints { private final HiveExpressionGenerator gen; private final HiveTable table; private final List columnsToBeAdded = new ArrayList<>(); - private boolean setPrimaryKey = false; + private boolean setPrimaryKey; public HiveTableGenerator(HiveGlobalState globalState, String tableName) { this.tableName = tableName; @@ -57,7 +57,7 @@ private SQLQueryAdapter create() { if (i != 0) { sb.append(", "); } - appendColumn(i, errors); + appendColumn(i); } sb.append(")"); @@ -71,17 +71,17 @@ private SQLQueryAdapter create() { return new SQLQueryAdapter(sb.toString(), errors, true, false); } - private void appendColumn(int columnId, ExpectedErrors errors) { + private void appendColumn(int columnId) { String columnName = DBMSCommon.createColumnName(columnId); sb.append(columnName); sb.append(" "); HiveDataType randType = HiveSchema.HiveDataType.getRandomType(); sb.append(randType); columnsToBeAdded.add(new HiveColumn(columnName, table, randType)); - appendColumnConstraint(randType, errors); + appendColumnConstraint(); } - private void appendColumnConstraint(HiveDataType type, ExpectedErrors errors) { + private void appendColumnConstraint() { /* * column_constraint_specification: : [ PRIMARY KEY|UNIQUE|NOT NULL|DEFAULT [default_value]|CHECK * [check_expression] ENABLE|DISABLE NOVALIDATE RELY/NORELY ] From 5d9d1d5ac747ed5a46eaf2f34ca79c869f04856d Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Fri, 23 May 2025 23:27:46 -0400 Subject: [PATCH 674/856] Add support for Alter Statistics and Set Statistics and remove force_parallel_mode --- src/sqlancer/postgres/PostgresProvider.java | 4 ++++ src/sqlancer/postgres/gen/PostgresCommon.java | 3 ++- .../postgres/gen/PostgresSetGenerator.java | 2 +- .../postgres/gen/PostgresStatisticsGenerator.java | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 2b2cb8585..347c7b6f2 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -99,6 +99,7 @@ public enum Action implements AbstractAction { }), // CREATE_STATISTICS(PostgresStatisticsGenerator::insert), // DROP_STATISTICS(PostgresStatisticsGenerator::remove), // + ALTER_STATISTICS(PostgresStatisticsGenerator::alter), // DELETE(PostgresDeleteGenerator::create), // DISCARD(PostgresDiscardGenerator::create), // DROP_INDEX(PostgresDropIndexGenerator::create), // @@ -150,6 +151,9 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case CREATE_STATISTICS: nrPerformed = r.getInteger(0, 5); break; + case ALTER_STATISTICS: + nrPerformed = r.getInteger(0, 2); + break; case DISCARD: case DROP_INDEX: nrPerformed = r.getInteger(0, 5); diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index a0cc21ae2..63bc885cb 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -434,6 +434,8 @@ private static void addTableConstraint(StringBuilder sb, PostgresTable table, Po } break; case EXCLUDE: + errors.add("exclusion constraints are not supported on partitioned tables"); + errors.add("unsupported EXCLUDE constraint with partition key definition"); sb.append("EXCLUDE "); sb.append("("); // TODO [USING index_method ] @@ -454,7 +456,6 @@ private static void addTableConstraint(StringBuilder sb, PostgresTable table, Po errors.add("exclusion constraints are not supported on partitioned tables"); errors.add("The exclusion operator must be related to the index operator class for the constraint"); errors.add("could not create exclusion constraint"); - // TODO: index parameters if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append("("); diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 440172c4b..38318b26b 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -112,7 +112,7 @@ private enum ConfigurationOption { JIT("jit", (r) -> Randomly.fromOptions(1, 0)), JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), - FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), + // FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), PLAN_CACHE_MODE("plan_cache_mode", (r) -> Randomly.fromOptions("auto", "force_generic_plan", "force_custom_plan")); diff --git a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java index e27f82800..9a82e29af 100644 --- a/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresStatisticsGenerator.java @@ -58,6 +58,20 @@ public static SQLQueryAdapter remove(PostgresGlobalState globalState) { return new SQLQueryAdapter(sb.toString(), true); } + public static SQLQueryAdapter alter(PostgresGlobalState globalState) { + StringBuilder sb = new StringBuilder("ALTER STATISTICS "); + PostgresTable randomTable = globalState.getSchema().getRandomTable(); + List statistics = randomTable.getStatistics(); + if (statistics.isEmpty()) { + throw new IgnoreMeException(); + } + PostgresStatisticsObject randomStatistic = Randomly.fromList(statistics); + sb.append(randomStatistic.getName()); + sb.append(" SET STATISTICS "); + sb.append(Randomly.getNotCachedInteger(-1, 10000)); // -1 means default + return new SQLQueryAdapter(sb.toString(), true); + } + private static String getNewStatisticsName(PostgresTable randomTable) { List statistics = randomTable.getStatistics(); int i = 0; From 3fe905a4294e7bbb0dcbf1e9d8d6cfa5aa1c0a28 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 24 May 2025 13:20:19 -0400 Subject: [PATCH 675/856] Add support for Alter View Rename Column in Postgres v13 --- .../gen/PostgresAlterTableGenerator.java | 45 ++++++++++++++++--- .../postgres/gen/PostgresSetGenerator.java | 2 +- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index f12927e90..b4ad39ecd 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -48,7 +48,7 @@ protected enum Action { SET_LOGGED_UNLOGGED, // NOT_OF, // OWNER_TO, // - REPLICA_IDENTITY + REPLICA_IDENTITY, ALTER_VIEW_RENAME_COLUMN // RENAME COLUMN old_name TO new_name (for views) } public PostgresAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, @@ -98,6 +98,20 @@ public List getActions(ExpectedErrors errors) { // make it more likely that the ALTER TABLE succeeds action = Randomly.subset(Randomly.smallNumber(), Action.values()); } + + // If this is a view, only allow view-compatible operations + if (randomTable.isView()) { + // Remove all table-specific operations for views + action.removeIf(a -> a != Action.ALTER_VIEW_RENAME_COLUMN); + // If no view operations remain, add the rename column action + if (action.isEmpty()) { + action.add(Action.ALTER_VIEW_RENAME_COLUMN); + } + } else { + // Remove view-specific actions if this is a table + action.remove(Action.ALTER_VIEW_RENAME_COLUMN); + } + if (randomTable.getColumns().size() == 1) { action.remove(Action.ALTER_TABLE_DROP_COLUMN); } @@ -120,14 +134,24 @@ public SQLQueryAdapter generate() { int i = 0; List action = getActions(errors); StringBuilder sb = new StringBuilder(); - sb.append("ALTER TABLE "); - if (Randomly.getBoolean()) { - sb.append(" ONLY"); - errors.add("cannot use ONLY for foreign key on partitioned table"); + + // Check if we're dealing with a view operation + boolean isViewOperation = action.contains(Action.ALTER_VIEW_RENAME_COLUMN); + + if (isViewOperation) { + sb.append("ALTER VIEW "); + } else { + sb.append("ALTER TABLE "); + if (Randomly.getBoolean()) { + sb.append(" ONLY"); + errors.add("cannot use ONLY for foreign key on partitioned table"); + } } + sb.append(" "); sb.append(randomTable.getName()); sb.append(" "); + for (Action a : action) { if (i++ != 0) { sb.append(", "); @@ -364,6 +388,17 @@ public SQLQueryAdapter generate() { errors.add("cannot use invalid index"); } break; + case ALTER_VIEW_RENAME_COLUMN: + sb.append("RENAME COLUMN "); + PostgresColumn columnToRename = randomTable.getRandomColumn(); + sb.append(columnToRename.getName()); + sb.append(" TO "); + sb.append("new_" + columnToRename.getName() + "_" + r.getInteger(1, 1000)); + errors.add("column does not exist"); + errors.add("column name already exists"); + errors.add("cannot rename column of view"); + errors.add("permission denied"); + break; default: throw new AssertionError(a); } diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 440172c4b..38318b26b 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -112,7 +112,7 @@ private enum ConfigurationOption { JIT("jit", (r) -> Randomly.fromOptions(1, 0)), JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), - FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), + // FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), PLAN_CACHE_MODE("plan_cache_mode", (r) -> Randomly.fromOptions("auto", "force_generic_plan", "force_custom_plan")); From d28c960a130267670d932c8705a408326fb5a3b6 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 24 May 2025 16:30:29 -0400 Subject: [PATCH 676/856] Add support for Drop Database Force --- src/sqlancer/postgres/PostgresProvider.java | 24 +++++++++++++++---- .../postgres/gen/PostgresSetGenerator.java | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 2b2cb8585..6f5e14019 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -275,12 +275,28 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE } Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + createDatabaseCommand = getCreateDatabaseCommand(globalState); - globalState.getState().logStatement(createDatabaseCommand); - try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + + // postgres 13+ supports force + if (Randomly.getBoolean()) { + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE FORCE IF EXISTS " + databaseName); + } catch (SQLException e) { + // If force fails, fall back to regular drop + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } + } + } else { + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName); + } } + + // Execute the create database command try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); } diff --git a/src/sqlancer/postgres/gen/PostgresSetGenerator.java b/src/sqlancer/postgres/gen/PostgresSetGenerator.java index 440172c4b..38318b26b 100644 --- a/src/sqlancer/postgres/gen/PostgresSetGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresSetGenerator.java @@ -112,7 +112,7 @@ private enum ConfigurationOption { JIT("jit", (r) -> Randomly.fromOptions(1, 0)), JOIN_COLLAPSE_LIMIT("join_collapse_limit", (r) -> r.getInteger(1, Integer.MAX_VALUE)), PARALLEL_LEADER_PARTICIPATION("parallel_leader_participation", (r) -> Randomly.fromOptions(1, 0)), - FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), + // FORCE_PARALLEL_MODE("force_parallel_mode", (r) -> Randomly.fromOptions("off", "on", "regress")), PLAN_CACHE_MODE("plan_cache_mode", (r) -> Randomly.fromOptions("auto", "force_generic_plan", "force_custom_plan")); From a513d9a4b16aec61d011d1cf0c4e00027976d481 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 24 May 2025 19:57:30 -0400 Subject: [PATCH 677/856] Add a list dedicated for view actions only --- .../gen/PostgresAlterTableGenerator.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index b4ad39ecd..33f1b202e 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -48,9 +48,14 @@ protected enum Action { SET_LOGGED_UNLOGGED, // NOT_OF, // OWNER_TO, // - REPLICA_IDENTITY, ALTER_VIEW_RENAME_COLUMN // RENAME COLUMN old_name TO new_name (for views) + REPLICA_IDENTITY, // RENAME COLUMN old_name TO new_name (for views) + ALTER_VIEW_RENAME_COLUMN // RENAME COLUMN old_name TO new_name (for views) } + private static final List VIEW_ACTIONS = List.of( + Action.ALTER_VIEW_RENAME_COLUMN + ); + public PostgresAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, boolean generateOnlyKnown) { this.randomTable = randomTable; @@ -101,15 +106,15 @@ public List getActions(ExpectedErrors errors) { // If this is a view, only allow view-compatible operations if (randomTable.isView()) { - // Remove all table-specific operations for views - action.removeIf(a -> a != Action.ALTER_VIEW_RENAME_COLUMN); - // If no view operations remain, add the rename column action + // Remove all non-view operations + action.removeIf(a -> !VIEW_ACTIONS.contains(a)); + // If no view operations remain, add a random view operation if (action.isEmpty()) { - action.add(Action.ALTER_VIEW_RENAME_COLUMN); + action.add(VIEW_ACTIONS.get(r.getInteger(0, VIEW_ACTIONS.size() - 1))); } } else { // Remove view-specific actions if this is a table - action.remove(Action.ALTER_VIEW_RENAME_COLUMN); + action.removeIf(VIEW_ACTIONS::contains); } if (randomTable.getColumns().size() == 1) { From 865cb58dae82ee5c73ae5383388d9d4e34403b66 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 31 May 2025 10:54:06 -0400 Subject: [PATCH 678/856] Add support for select with ties --- .../postgres/ast/PostgresOrderByTerm.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java index 96debf741..76257215b 100644 --- a/src/sqlancer/postgres/ast/PostgresOrderByTerm.java +++ b/src/sqlancer/postgres/ast/PostgresOrderByTerm.java @@ -7,6 +7,8 @@ public class PostgresOrderByTerm implements PostgresExpression { private final PostgresExpression expr; private final PostgresOrder order; + private final int limit; + private final boolean ties; public enum PostgresOrder { ASC, DESC; @@ -22,6 +24,15 @@ public PostgresOrderByTerm(PostgresExpression expr, PostgresOrder order) { } this.expr = expr; this.order = order; + + if (Randomly.getBooleanWithRatherLowProbability()) { + this.limit = (int) Randomly.getPositiveOrZeroNonCachedInteger(); + this.ties = true; + } else { + this.limit = 0; + this.ties = false; + } + } // Constructor for window functions, might be removed in the future to have only one constructor @@ -31,6 +42,8 @@ public PostgresOrderByTerm(PostgresExpression expr, boolean ascending) { } this.expr = expr; this.order = ascending ? PostgresOrder.ASC : PostgresOrder.DESC; + this.limit = 0; + this.ties = false; } public PostgresExpression getExpr() { @@ -57,6 +70,10 @@ public PostgresDataType getExpressionType() { @Override public String toString() { - return String.format("%s %s", expr, order); + if (ties) { + return String.format("%s %s FETCH FIRST %d WITH TIES", expr, order, limit); + } else { + return String.format("%s %s", expr, order); + } } } From ec6f8a799f7e4778811dfa275f43e8ea3a849835 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 31 May 2025 21:22:34 -0400 Subject: [PATCH 679/856] Add support for alter column drop expression --- .../postgres/gen/PostgresAlterTableGenerator.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index f12927e90..ce8674ac9 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -34,6 +34,7 @@ protected enum Action { ALTER_COLUMN_SET_ATTRIBUTE_OPTION, // ALTER [ COLUMN ] column SET ( attribute_option = value [, ... ] ) ALTER_COLUMN_RESET_ATTRIBUTE_OPTION, // ALTER [ COLUMN ] column RESET ( attribute_option [, ... ] ) ALTER_COLUMN_SET_STORAGE, // ALTER [ COLUMN ] column SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN } + ALTER_COLUMN_DROP_EXPRESSION, // ALTER [ COLUMN ] column DROP EXPRESSION [ IF EXISTS ] ADD_TABLE_CONSTRAINT, // ADD table_constraint [ NOT VALID ] ADD_TABLE_CONSTRAINT_USING_INDEX, // ADD table_constraint_using_index VALIDATE_CONSTRAINT, // VALIDATE CONSTRAINT constraint_name @@ -250,6 +251,19 @@ public SQLQueryAdapter generate() { errors.add("can only have storage"); errors.add("is an identity column"); break; + case ALTER_COLUMN_DROP_EXPRESSION: + alterColumn(randomTable, sb); + sb.append("DROP EXPRESSION"); + if (Randomly.getBoolean()) { + sb.append(" IF EXISTS"); + } + errors.add("is not a generated column"); + errors.add("is not a stored generated column"); + errors.add("cannot drop expression from inherited column"); + errors.add("cannot drop generation expression from inherited column"); + errors.add("must be applied to child tables too"); + errors.add("cannot drop expression from column"); + break; case ADD_TABLE_CONSTRAINT: sb.append("ADD "); sb.append("CONSTRAINT " + r.getAlphabeticChar() + " "); From afbdfe1b184602542f8d9bae5fe8fbdb60289fb8 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sat, 31 May 2025 21:54:39 -0400 Subject: [PATCH 680/856] Add support for Create DB with Locale --- src/sqlancer/postgres/PostgresProvider.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 2b2cb8585..15e95b064 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -340,9 +340,13 @@ private String getCreateDatabaseCommand(PostgresGlobalState state) { sb.append(Randomly.fromOptions("utf8")); sb.append("' "); } - for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { - if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { - sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + if (Randomly.getBoolean() && !state.getCollates().isEmpty()) { + sb.append(String.format(" LOCALE = '%s' ", Randomly.fromList(state.getCollates()))); + } else { + for (String lc : Arrays.asList("LC_COLLATE", "LC_CTYPE")) { + if (!state.getCollates().isEmpty() && Randomly.getBoolean()) { + sb.append(String.format(" %s = '%s'", lc, Randomly.fromList(state.getCollates()))); + } } } sb.append(" TEMPLATE template0"); From dad655171c07643e64f34028b0157a1632e1fd9b Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 1 Jun 2025 21:52:07 -0400 Subject: [PATCH 681/856] Add support for tablespace generator --- src/sqlancer/postgres/PostgresProvider.java | 7 ++- .../gen/PostgresTableSpaceGenerator.java | 49 +++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 2b2cb8585..5c2897208 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -51,6 +51,7 @@ import sqlancer.postgres.gen.PostgresUpdateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; import sqlancer.postgres.gen.PostgresViewGenerator; +import sqlancer.postgres.gen.PostgresTableSpaceGenerator; // EXISTS // IN @@ -125,7 +126,8 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(PostgresViewGenerator::create); + CREATE_VIEW(PostgresViewGenerator::create), + CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); private final SQLQueryProvider sqlQueryProvider; @@ -186,6 +188,9 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case CREATE_VIEW: nrPerformed = r.getInteger(0, 2); break; + case CREATE_TABLESPACE: + nrPerformed = r.getInteger(0, 2); + break; case UPDATE: nrPerformed = r.getInteger(0, 10); break; diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java new file mode 100644 index 000000000..2f0065408 --- /dev/null +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -0,0 +1,49 @@ +package sqlancer.postgres.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; + +public class PostgresTableSpaceGenerator { + + private final ExpectedErrors errors = new ExpectedErrors(); + private final PostgresGlobalState globalState; + + public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; + errors.add("ERROR: directory \"/tmp/postgresql/tablespace1\" does not exist"); + errors.add("ERROR: directory \"/tmp/postgresql/tablespace2\" does not exist"); + errors.add("ERROR: directory \"/tmp/postgresql/tablespace3\" does not exist"); + errors.add("ERROR: directory \"/tmp/postgresql/tablespace4\" does not exist"); + errors.add("ERROR: directory \"/tmp/postgresql/tablespace5\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace1\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace2\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace3\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace4\" does not exist"); + errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace5\" does not exist"); + errors.add("ERROR: must be a directory"); + errors.add("ERROR: permission denied"); + errors.add("ERROR: already exists"); + errors.add("ERROR: is not empty"); + errors.add("ERROR: cannot be created because system does not support tablespaces"); + } + + public static SQLQueryAdapter generate(PostgresGlobalState globalState) { + return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); + } + + private SQLQueryAdapter generateTableSpace() { + StringBuilder sb = new StringBuilder(); + int tableSpaceNum = globalState.getRandomly().getInteger(1, 5); // Use the instance method + + // CREATE TABLESPACE syntax + sb.append("CREATE TABLESPACE "); + sb.append("tablespace"); + sb.append(tableSpaceNum); + sb.append(" LOCATION '/tmp/postgresql/tablespace"); + sb.append(tableSpaceNum); + sb.append("'"); + + return new SQLQueryAdapter(sb.toString(), errors); + } +} From 5766078e2715f861655b7707cd5b734b89550b37 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 1 Jun 2025 22:06:09 -0400 Subject: [PATCH 682/856] Add a clearer execution statement --- src/sqlancer/postgres/PostgresProvider.java | 34 ++++++++++++--------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 6f5e14019..3162a03fb 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -276,27 +276,31 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - createDatabaseCommand = getCreateDatabaseCommand(globalState); - - // postgres 13+ supports force + String dropCommand = "DROP DATABASE"; if (Randomly.getBoolean()) { - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE FORCE IF EXISTS " + databaseName); - } catch (SQLException e) { - // If force fails, fall back to regular drop + dropCommand += " FORCE"; + } + dropCommand += " IF EXISTS " + databaseName; + + globalState.getState().logStatement(dropCommand + ";"); + try (Statement s = con.createStatement()) { + s.execute(dropCommand); + } catch (SQLException e) { + // If force fails, fall back to regular drop + if (dropCommand.contains("FORCE")) { + String fallbackDrop = "DROP DATABASE IF EXISTS " + databaseName; + globalState.getState().logStatement(fallbackDrop + ";"); try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + s.execute(fallbackDrop); } - } - } else { - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + } else { + throw e; } } - // Execute the create database command + // Create database section + createDatabaseCommand = getCreateDatabaseCommand(globalState); + globalState.getState().logStatement(createDatabaseCommand + ";"); try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); } From 5868650179c66fc23026e22e517b9438579381d0 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Tue, 3 Jun 2025 23:32:02 -0400 Subject: [PATCH 683/856] Fix AlterTableGenerator format issue --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index c8842f0a4..6e0c436ee 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -53,9 +53,7 @@ protected enum Action { ALTER_VIEW_RENAME_COLUMN // RENAME COLUMN old_name TO new_name (for views) } - private static final List VIEW_ACTIONS = List.of( - Action.ALTER_VIEW_RENAME_COLUMN - ); + private static final List VIEW_ACTIONS = List.of(Action.ALTER_VIEW_RENAME_COLUMN); public PostgresAlterTableGenerator(PostgresTable randomTable, PostgresGlobalState globalState, boolean generateOnlyKnown) { From 38fcee824b8447660b7a078b7282430ab8c2ac68 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 5 Jun 2025 10:54:00 +0800 Subject: [PATCH 684/856] fix issue #1226 that incorrect constraint make the code can not generate the generated columns with the default configuration for SQLite --- src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java index 1d045982b..4e83fbca5 100644 --- a/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java +++ b/src/sqlancer/sqlite3/gen/SQLite3ColumnBuilder.java @@ -51,7 +51,7 @@ public String createColumn(String columnName, SQLite3GlobalState globalState, Li if (Randomly.getBooleanWithRatherLowProbability()) { List constraints = Randomly.subset(Constraints.values()); if (!Randomly.getBooleanWithSmallProbability() - || globalState.getDbmsSpecificOptions().testGeneratedColumns) { + || !globalState.getDbmsSpecificOptions().testGeneratedColumns) { constraints.remove(Constraints.GENERATED_AS); } if (constraints.contains(Constraints.GENERATED_AS)) { From 25b51b6ca4f232f8b5a0d774b2668ce7d64171fd Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 5 Jun 2025 13:52:29 +0800 Subject: [PATCH 685/856] fix issue #1239 --- .../sqlite3/oracle/SQLite3RandomQuerySynthesizer.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java index d3d7ded31..f8f53ae3b 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3RandomQuerySynthesizer.java @@ -37,10 +37,11 @@ public static SQLite3Expression generate(SQLite3GlobalState globalState, int siz SQLite3Tables targetTables = s.getRandomTableNonEmptyTables(); List expressions = new ArrayList<>(); SQLite3ExpressionGenerator gen = new SQLite3ExpressionGenerator(globalState) - .setColumns(s.getTables().getColumns()); - SQLite3ExpressionGenerator whereClauseGen = new SQLite3ExpressionGenerator(globalState); + .setColumns(targetTables.getColumns()); + SQLite3ExpressionGenerator whereClauseGen = new SQLite3ExpressionGenerator(globalState) + .setColumns(targetTables.getColumns()); SQLite3ExpressionGenerator aggregateGen = new SQLite3ExpressionGenerator(globalState) - .setColumns(s.getTables().getColumns()).allowAggregateFunctions(); + .setColumns(targetTables.getColumns()).allowAggregateFunctions(); // SELECT SQLite3Select select = new SQLite3Select(); From ebda7309b84389209f9ba2432cdb823da8cfd97b Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 8 Jun 2025 20:19:33 -0400 Subject: [PATCH 686/856] Add boolean flag for drop db force --- src/sqlancer/postgres/PostgresProvider.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 3162a03fb..3a2b6b357 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -277,7 +277,8 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); String dropCommand = "DROP DATABASE"; - if (Randomly.getBoolean()) { + boolean forceDrop = Randomly.getBoolean(); + if (forceDrop) { dropCommand += " FORCE"; } dropCommand += " IF EXISTS " + databaseName; @@ -287,7 +288,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE s.execute(dropCommand); } catch (SQLException e) { // If force fails, fall back to regular drop - if (dropCommand.contains("FORCE")) { + if (forceDrop) { String fallbackDrop = "DROP DATABASE IF EXISTS " + databaseName; globalState.getState().logStatement(fallbackDrop + ";"); try (Statement s = con.createStatement()) { From 7e0b0f4d33fc5a8e02f2abb763c3145124e2174d Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 8 Jun 2025 20:58:23 -0400 Subject: [PATCH 687/856] Add regex and OS option for TableSpaceGenerator --- src/sqlancer/postgres/PostgresOptions.java | 41 ++++++++++++++++++- src/sqlancer/postgres/PostgresProvider.java | 5 +-- .../gen/PostgresTableSpaceGenerator.java | 40 ++++++++++-------- 3 files changed, 66 insertions(+), 20 deletions(-) diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 659dc4ff6..e3b93269d 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -1,5 +1,6 @@ package sqlancer.postgres; +import java.io.File; import java.util.Arrays; import java.util.List; @@ -13,6 +14,7 @@ public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; + private static final boolean DEFAULT_TEST_TABLESPACES = determineDefaultTablespaceSupport(); @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) public boolean allowBulkInsert; @@ -23,6 +25,12 @@ public class PostgresOptions implements DBMSSpecificOptions getTestOracleFactory() { return oracle; } - } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 5c2897208..881239d85 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -46,12 +46,12 @@ import sqlancer.postgres.gen.PostgresSetGenerator; import sqlancer.postgres.gen.PostgresStatisticsGenerator; import sqlancer.postgres.gen.PostgresTableGenerator; +import sqlancer.postgres.gen.PostgresTableSpaceGenerator; import sqlancer.postgres.gen.PostgresTransactionGenerator; import sqlancer.postgres.gen.PostgresTruncateGenerator; import sqlancer.postgres.gen.PostgresUpdateGenerator; import sqlancer.postgres.gen.PostgresVacuumGenerator; import sqlancer.postgres.gen.PostgresViewGenerator; -import sqlancer.postgres.gen.PostgresTableSpaceGenerator; // EXISTS // IN @@ -126,8 +126,7 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(PostgresViewGenerator::create), - CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); + CREATE_VIEW(PostgresViewGenerator::create), CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); private final SQLQueryProvider sqlQueryProvider; diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java index 2f0065408..89f47cef2 100644 --- a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -3,24 +3,16 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; public class PostgresTableSpaceGenerator { - + private final ExpectedErrors errors = new ExpectedErrors(); private final PostgresGlobalState globalState; public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { this.globalState = globalState; - errors.add("ERROR: directory \"/tmp/postgresql/tablespace1\" does not exist"); - errors.add("ERROR: directory \"/tmp/postgresql/tablespace2\" does not exist"); - errors.add("ERROR: directory \"/tmp/postgresql/tablespace3\" does not exist"); - errors.add("ERROR: directory \"/tmp/postgresql/tablespace4\" does not exist"); - errors.add("ERROR: directory \"/tmp/postgresql/tablespace5\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace1\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace2\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace3\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace4\" does not exist"); - errors.add("ERROR: tablespace directory \"/tmp/postgresql/tablespace5\" does not exist"); + errors.addRegexString("ERROR: (?:tablespace )?directory \".*[\\\\/]tablespace[1-5]\" does not exist"); errors.add("ERROR: must be a directory"); errors.add("ERROR: permission denied"); errors.add("ERROR: already exists"); @@ -29,21 +21,37 @@ public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { } public static SQLQueryAdapter generate(PostgresGlobalState globalState) { + // Skip tablespace generation if the option is disabled + PostgresOptions options = globalState.getDbmsSpecificOptions(); + if (!options.testTablespaces) { + return null; + } return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); } private SQLQueryAdapter generateTableSpace() { StringBuilder sb = new StringBuilder(); - int tableSpaceNum = globalState.getRandomly().getInteger(1, 5); // Use the instance method - + int tableSpaceNum = globalState.getRandomly().getInteger(1, 5); + // CREATE TABLESPACE syntax sb.append("CREATE TABLESPACE "); sb.append("tablespace"); sb.append(tableSpaceNum); - sb.append(" LOCATION '/tmp/postgresql/tablespace"); - sb.append(tableSpaceNum); + sb.append(" LOCATION '"); + + // Get the base path from options and append the tablespace number + PostgresOptions options = globalState.getDbmsSpecificOptions(); + String path = options.tablespacePath + tableSpaceNum; + + // Convert backslashes to forward slashes for PostgreSQL + path = path.replace('\\', '/'); + + // Escape single quotes in the path + path = path.replace("'", "''"); + + sb.append(path); sb.append("'"); - + return new SQLQueryAdapter(sb.toString(), errors); } } From 4e1ec862893cec86df771b5fca89e6d56b322867 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Tue, 10 Jun 2025 15:34:19 +0800 Subject: [PATCH 688/856] Bump SQLite from 3.47.2.0 to 3.49.1.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 83068f498..734b44f5d 100644 --- a/pom.xml +++ b/pom.xml @@ -299,7 +299,7 @@ org.xerial sqlite-jdbc - 3.47.2.0 + 3.49.1.0 mysql From 22e928ad1dca97d8360d24b93e38e352339b8501 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Wed, 11 Jun 2025 21:58:19 +0800 Subject: [PATCH 689/856] Bump DuckDB from 1.2.0 to 1.3.0.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 83068f498..83ebf893d 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.duckdb duckdb_jdbc - 1.2.0 + 1.3.0.0 com.facebook.presto From ffef7d12997476a9b55b07d67e4140286c947bd6 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Thu, 12 Jun 2025 11:21:56 +0800 Subject: [PATCH 690/856] Add the expected error for COLLATE on non-varchar Statement: SELECT t22.rowid FROM t22, t0 WHERE t0.c1 ORDER BY t22.c0, t22.rowid, ((1) COLLATE NOACCENT) DESC; Binder Error: COLLATE can only be applied to varchar columns --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index f5cf3eb94..394de69d4 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -44,6 +44,7 @@ public static List getExpressionErrors() { // collate errors.add("Cannot combine types with different collation!"); errors.add("collations are only supported for type varchar"); + errors.add("COLLATE can only be applied to varchar columns"); errors.add("Like pattern must not end with escape character!"); // LIKE From 495ad713bbca745e05a21c4b3cfd0fee5b93bc66 Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Tue, 17 Jun 2025 22:41:17 -0400 Subject: [PATCH 691/856] Refactored the module --- src/sqlancer/postgres/PostgresOptions.java | 46 ++++++++++++++++++- src/sqlancer/postgres/PostgresProvider.java | 3 +- .../gen/PostgresTableSpaceGenerator.java | 11 ++--- 3 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index e3b93269d..62a115cf4 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -14,7 +14,7 @@ public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; - private static final boolean DEFAULT_TEST_TABLESPACES = determineDefaultTablespaceSupport(); + private static Boolean defaultTestTablespaces = null; @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) public boolean allowBulkInsert; @@ -26,7 +26,7 @@ public class PostgresOptions implements DBMSSpecificOptions getTestOracleFactory() { return oracle; } + + public String getTablespacePath() { + if (tablespacePath == null || tablespacePath.trim().isEmpty()) { + throw new AssertionError("Tablespace path is null or empty. Please configure --tablespace-path"); + } + + File path = new File(tablespacePath); + + // Check if the directory exists or can be created + if (!path.exists()) { + if (!path.mkdirs()) { + throw new AssertionError("Cannot create tablespace directory: " + tablespacePath + + ". Please ensure the parent directory exists and you have write permissions."); + } + } + + // Check if it's actually a directory + if (!path.isDirectory()) { + throw new AssertionError("Tablespace path is not a directory: " + tablespacePath); + } + + // Check write permissions + if (!path.canWrite()) { + throw new AssertionError("No write permissions for tablespace directory: " + tablespacePath + + ". Please ensure you have write permissions to this directory."); + } + + return tablespacePath; + } + + public boolean isTestTablespaces() { + // If the user explicitly set the value via command line, use that + // Otherwise, use the OS-dependent default + return testTablespaces || getDefaultTablespaceSupport(); + } + + private static boolean getDefaultTablespaceSupport() { + if (defaultTestTablespaces == null) { + defaultTestTablespaces = determineDefaultTablespaceSupport(); + } + return defaultTestTablespaces; + } } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 881239d85..c3f161a7e 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -126,7 +126,8 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(PostgresViewGenerator::create), CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); + CREATE_VIEW(PostgresViewGenerator::create), + CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); private final SQLQueryProvider sqlQueryProvider; diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java index 89f47cef2..6c44043b8 100644 --- a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -12,8 +12,7 @@ public class PostgresTableSpaceGenerator { public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { this.globalState = globalState; - errors.addRegexString("ERROR: (?:tablespace )?directory \".*[\\\\/]tablespace[1-5]\" does not exist"); - errors.add("ERROR: must be a directory"); + errors.addRegexString("ERROR: (?:tablespace )?directory \".*[\\\\/]tablespace\\d+\" does not exist"); errors.add("ERROR: permission denied"); errors.add("ERROR: already exists"); errors.add("ERROR: is not empty"); @@ -23,7 +22,7 @@ public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { public static SQLQueryAdapter generate(PostgresGlobalState globalState) { // Skip tablespace generation if the option is disabled PostgresOptions options = globalState.getDbmsSpecificOptions(); - if (!options.testTablespaces) { + if (!options.isTestTablespaces()) { return null; } return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); @@ -31,7 +30,7 @@ public static SQLQueryAdapter generate(PostgresGlobalState globalState) { private SQLQueryAdapter generateTableSpace() { StringBuilder sb = new StringBuilder(); - int tableSpaceNum = globalState.getRandomly().getInteger(1, 5); + int tableSpaceNum = globalState.getRandomly().getInteger(1, Integer.MAX_VALUE); // CREATE TABLESPACE syntax sb.append("CREATE TABLESPACE "); @@ -39,9 +38,9 @@ private SQLQueryAdapter generateTableSpace() { sb.append(tableSpaceNum); sb.append(" LOCATION '"); - // Get the base path from options and append the tablespace number + // Get the validated base path from options and append the tablespace number PostgresOptions options = globalState.getDbmsSpecificOptions(); - String path = options.tablespacePath + tableSpaceNum; + String path = options.getTablespacePath() + tableSpaceNum; // Convert backslashes to forward slashes for PostgreSQL path = path.replace('\\', '/'); From 202bd3da76c5045d8f741bf5cb49364246d4590e Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Tue, 17 Jun 2025 22:48:16 -0400 Subject: [PATCH 692/856] Reformat the code --- src/sqlancer/postgres/PostgresOptions.java | 31 +++++++++++---------- src/sqlancer/postgres/PostgresProvider.java | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/sqlancer/postgres/PostgresOptions.java b/src/sqlancer/postgres/PostgresOptions.java index 62a115cf4..ebef13a16 100644 --- a/src/sqlancer/postgres/PostgresOptions.java +++ b/src/sqlancer/postgres/PostgresOptions.java @@ -14,7 +14,7 @@ public class PostgresOptions implements DBMSSpecificOptions { public static final String DEFAULT_HOST = "localhost"; public static final int DEFAULT_PORT = 5432; - private static Boolean defaultTestTablespaces = null; + private static Boolean defaultTestTablespaces; @Parameter(names = "--bulk-insert", description = "Specifies whether INSERT statements should be issued in bulk", arity = 1) public boolean allowBulkInsert; @@ -22,11 +22,14 @@ public class PostgresOptions implements DBMSSpecificOptions oracle = Arrays.asList(PostgresOracleFactory.QUERY_PARTITIONING); + @Parameter(names = "--connection-timeout", description = "Timeout in seconds for connecting to the server", arity = 1) + public int connectionTimeoutInSeconds; + @Parameter(names = "--test-collations", description = "Specifies whether to test different collations", arity = 1) public boolean testCollations = true; @Parameter(names = "--test-tablespaces", description = "Specifies whether to test tablespace creation (default is OS-dependent)", arity = 1) - public boolean testTablespaces = false; + public boolean testTablespaces; @Parameter(names = "--tablespace-path", description = "Base path for tablespace directories (default is OS-dependent)", arity = 1) public String tablespacePath = getDefaultTablespacePath(); @@ -76,31 +79,29 @@ public List getTestOracleFactory() { } public String getTablespacePath() { - if (tablespacePath == null || tablespacePath.trim().isEmpty()) { + if (tablespacePath == null || tablespacePath.isBlank()) { throw new AssertionError("Tablespace path is null or empty. Please configure --tablespace-path"); } - + File path = new File(tablespacePath); - + // Check if the directory exists or can be created - if (!path.exists()) { - if (!path.mkdirs()) { - throw new AssertionError("Cannot create tablespace directory: " + tablespacePath + - ". Please ensure the parent directory exists and you have write permissions."); - } + if (!path.exists() && !path.mkdirs()) { + throw new AssertionError("Cannot create tablespace directory: " + tablespacePath + + ". Please ensure the parent directory exists and you have write permissions."); } - + // Check if it's actually a directory if (!path.isDirectory()) { throw new AssertionError("Tablespace path is not a directory: " + tablespacePath); } - + // Check write permissions if (!path.canWrite()) { - throw new AssertionError("No write permissions for tablespace directory: " + tablespacePath + - ". Please ensure you have write permissions to this directory."); + throw new AssertionError("No write permissions for tablespace directory: " + tablespacePath + + ". Please ensure you have write permissions to this directory."); } - + return tablespacePath; } diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index c3f161a7e..5b22cd83d 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -126,7 +126,7 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // - CREATE_VIEW(PostgresViewGenerator::create), + CREATE_VIEW(PostgresViewGenerator::create), // CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); private final SQLQueryProvider sqlQueryProvider; From 0cc62d25ef318aa4b6ea5cf7336cbaabaa736420 Mon Sep 17 00:00:00 2001 From: HDQ Date: Wed, 9 Jul 2025 12:30:32 +0800 Subject: [PATCH 693/856] Add the SQLQueryError class for DQE and its test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The core idea of DQE is that the SELECT, UPDATE and DELETE queries with the same predicate φ should access the same rows. If these queries access different rows, DQE reveals a potential logic bug in the target DBMS. SQLQueryError class is used to compare the errors --- src/sqlancer/common/query/SQLQueryError.java | 116 ++++++++++++++++++ .../common/query/SQLQueryErrorTest.java | 93 ++++++++++++++ 2 files changed, 209 insertions(+) create mode 100644 src/sqlancer/common/query/SQLQueryError.java create mode 100644 test/sqlancer/common/query/SQLQueryErrorTest.java diff --git a/src/sqlancer/common/query/SQLQueryError.java b/src/sqlancer/common/query/SQLQueryError.java new file mode 100644 index 000000000..cde2a40fb --- /dev/null +++ b/src/sqlancer/common/query/SQLQueryError.java @@ -0,0 +1,116 @@ +package sqlancer.common.query; + +public class SQLQueryError implements Comparable { + + public enum ErrorLevel { + WARNING, ERROR + } + + private ErrorLevel level; + private int code; + private String message; + + public void setLevel(String level) { + // value of is case-sensitive + this.level = ErrorLevel.valueOf(level.toUpperCase()); + } + + public void setCode(int code) { + this.code = code; + } + + public void setMessage(String message) { + this.message = message; + } + + public ErrorLevel getLevel() { + return level; + } + + public int getCode() { + return code; + } + + public String getMessage() { + return message; + } + + public boolean hasSameLevel(SQLQueryError that) { + if (level == null && that.getLevel() == null) { + return true; + } else if (level != null && level.equals(that.getLevel())) { + return true; + } else { + return false; + } + } + + public boolean hasSameCodeAndMessage(SQLQueryError that) { + if (code != that.getCode()) { + return false; + } + if (message == null && that.getMessage() == null) { + return true; + } else if (message != null && message.equals(that.getMessage())) { + return true; + } else { + return false; + } + } + + @Override + public boolean equals(Object that) { + if (that == null) { + return false; + } + if (that instanceof SQLQueryError) { + SQLQueryError thatError = (SQLQueryError) that; + if (hasSameLevel(thatError) && hasSameCodeAndMessage(thatError)) { + return true; + } + } + return false; + } + + @Override + public String toString() { + return String.format("Level: %s; Code: %d; Message: %s.", level, code, message); + } + + @Override + public int compareTo(SQLQueryError that) { + if (code < that.getCode()) { + return -1; + } else if (code > that.getCode()) { + return 1; + } + + if (level == null && that.getLevel() != null) { + return -1; + } + if (level != null && that.getLevel() == null) { + return 1; + } + if (level != null && that.getLevel() != null) { + int res = level.compareTo(that.getLevel()); + if (res != 0) { + return res; + } + } + + if (message == null && that.getMessage() != null) { + return -1; + } + if (message != null && that.getMessage() == null) { + return 1; + } + if (message != null && that.getMessage() != null) { + int res = message.compareTo(that.getMessage()); + if (res != 0) { + return res; + } + } + + return 0; + } +} \ No newline at end of file diff --git a/test/sqlancer/common/query/SQLQueryErrorTest.java b/test/sqlancer/common/query/SQLQueryErrorTest.java new file mode 100644 index 000000000..0b80de176 --- /dev/null +++ b/test/sqlancer/common/query/SQLQueryErrorTest.java @@ -0,0 +1,93 @@ +package sqlancer.common.query; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class SQLQueryErrorTest { + @Test + public void testSettersAndGetters() { + SQLQueryError error = new SQLQueryError(); + error.setLevel("error"); + error.setCode(123); + error.setMessage("Test message"); + assertEquals(SQLQueryError.ErrorLevel.ERROR, error.getLevel()); + assertEquals(123, error.getCode()); + assertEquals("Test message", error.getMessage()); + } + + @Test + public void testHasSameLevel() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setLevel("warning"); + e2.setLevel("warning"); + assertTrue(e1.hasSameLevel(e2)); + e2.setLevel("error"); + assertFalse(e1.hasSameLevel(e2)); + } + + @Test + public void testHasSameCodeAndMessage() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setCode(1); + e2.setCode(1); + e1.setMessage("msg"); + e2.setMessage("msg"); + assertTrue(e1.hasSameCodeAndMessage(e2)); + e2.setCode(2); + assertFalse(e1.hasSameCodeAndMessage(e2)); + e2.setCode(1); + e2.setMessage("other"); + assertFalse(e1.hasSameCodeAndMessage(e2)); + } + + @Test + public void testEquals() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setLevel("error"); + e1.setCode(1); + e1.setMessage("msg"); + e2.setLevel("error"); + e2.setCode(1); + e2.setMessage("msg"); + assertEquals(e1, e2); + e2.setLevel("warning"); + assertNotEquals(e1, e2); + } + + @Test + public void testToString() { + SQLQueryError e = new SQLQueryError(); + e.setLevel("error"); + e.setCode(1); + e.setMessage("msg"); + String str = e.toString(); + assertTrue(str.contains("Level: ERROR")); + assertTrue(str.contains("Code: 1")); + assertTrue(str.contains("Message: msg")); + } + + @Test + public void testCompareTo() { + SQLQueryError e1 = new SQLQueryError(); + SQLQueryError e2 = new SQLQueryError(); + e1.setCode(1); + e2.setCode(2); + assertTrue(e1.compareTo(e2) < 0); + e2.setCode(1); + e1.setLevel("error"); + e2.setLevel("warning"); + assertTrue(e1.compareTo(e2) > 0 || e1.compareTo(e2) < 0); + e2.setLevel("error"); + e1.setMessage("a"); + e2.setMessage("b"); + assertTrue(e1.compareTo(e2) < 0); + } +} + From cc561e9feb760ffd7d6e11ff3626abc97c08fbf9 Mon Sep 17 00:00:00 2001 From: HDQ Date: Tue, 15 Jul 2025 21:47:47 +0800 Subject: [PATCH 694/856] Promote ErrorLevel and compare methods --- src/sqlancer/common/query/SQLQueryError.java | 57 ++++++++----------- .../common/query/SQLQueryErrorTest.java | 22 +++---- 2 files changed, 36 insertions(+), 43 deletions(-) diff --git a/src/sqlancer/common/query/SQLQueryError.java b/src/sqlancer/common/query/SQLQueryError.java index cde2a40fb..4ab253616 100644 --- a/src/sqlancer/common/query/SQLQueryError.java +++ b/src/sqlancer/common/query/SQLQueryError.java @@ -10,9 +10,8 @@ public enum ErrorLevel { private int code; private String message; - public void setLevel(String level) { - // value of is case-sensitive - this.level = ErrorLevel.valueOf(level.toUpperCase()); + public void setLevel(ErrorLevel level) { + this.level = level; } public void setCode(int code) { @@ -36,12 +35,10 @@ public String getMessage() { } public boolean hasSameLevel(SQLQueryError that) { - if (level == null && that.getLevel() == null) { - return true; - } else if (level != null && level.equals(that.getLevel())) { - return true; + if (level == null) { + return that.getLevel() == null; } else { - return false; + return level.equals(that.getLevel()); } } @@ -49,12 +46,10 @@ public boolean hasSameCodeAndMessage(SQLQueryError that) { if (code != that.getCode()) { return false; } - if (message == null && that.getMessage() == null) { - return true; - } else if (message != null && message.equals(that.getMessage())) { - return true; + if (message == null) { + return that.getMessage() == null; } else { - return false; + return message.equals(that.getMessage()); } } @@ -65,9 +60,7 @@ public boolean equals(Object that) { } if (that instanceof SQLQueryError) { SQLQueryError thatError = (SQLQueryError) that; - if (hasSameLevel(thatError) && hasSameCodeAndMessage(thatError)) { - return true; - } + return hasSameLevel(thatError) && hasSameCodeAndMessage(thatError); } return false; } @@ -87,27 +80,27 @@ public int compareTo(SQLQueryError that) { if (level == null && that.getLevel() != null) { return -1; - } - if (level != null && that.getLevel() == null) { - return 1; - } - if (level != null && that.getLevel() != null) { - int res = level.compareTo(that.getLevel()); - if (res != 0) { - return res; + } else { + if (that.getLevel() == null) { + return 1; + } else { + int res = level.compareTo(that.getLevel()); + if (res != 0) { + return res; + } } } if (message == null && that.getMessage() != null) { return -1; - } - if (message != null && that.getMessage() == null) { - return 1; - } - if (message != null && that.getMessage() != null) { - int res = message.compareTo(that.getMessage()); - if (res != 0) { - return res; + } else { + if (that.getMessage() == null) { + return 1; + } else { + int res = message.compareTo(that.getMessage()); + if (res != 0) { + return res; + } } } diff --git a/test/sqlancer/common/query/SQLQueryErrorTest.java b/test/sqlancer/common/query/SQLQueryErrorTest.java index 0b80de176..e0065788e 100644 --- a/test/sqlancer/common/query/SQLQueryErrorTest.java +++ b/test/sqlancer/common/query/SQLQueryErrorTest.java @@ -11,7 +11,7 @@ public class SQLQueryErrorTest { @Test public void testSettersAndGetters() { SQLQueryError error = new SQLQueryError(); - error.setLevel("error"); + error.setLevel(SQLQueryError.ErrorLevel.ERROR); error.setCode(123); error.setMessage("Test message"); assertEquals(SQLQueryError.ErrorLevel.ERROR, error.getLevel()); @@ -23,10 +23,10 @@ public void testSettersAndGetters() { public void testHasSameLevel() { SQLQueryError e1 = new SQLQueryError(); SQLQueryError e2 = new SQLQueryError(); - e1.setLevel("warning"); - e2.setLevel("warning"); + e1.setLevel(SQLQueryError.ErrorLevel.WARNING); + e2.setLevel(SQLQueryError.ErrorLevel.WARNING); assertTrue(e1.hasSameLevel(e2)); - e2.setLevel("error"); + e2.setLevel(SQLQueryError.ErrorLevel.ERROR); assertFalse(e1.hasSameLevel(e2)); } @@ -50,21 +50,21 @@ public void testHasSameCodeAndMessage() { public void testEquals() { SQLQueryError e1 = new SQLQueryError(); SQLQueryError e2 = new SQLQueryError(); - e1.setLevel("error"); + e1.setLevel(SQLQueryError.ErrorLevel.ERROR); e1.setCode(1); e1.setMessage("msg"); - e2.setLevel("error"); + e2.setLevel(SQLQueryError.ErrorLevel.ERROR); e2.setCode(1); e2.setMessage("msg"); assertEquals(e1, e2); - e2.setLevel("warning"); + e2.setLevel(SQLQueryError.ErrorLevel.WARNING); assertNotEquals(e1, e2); } @Test public void testToString() { SQLQueryError e = new SQLQueryError(); - e.setLevel("error"); + e.setLevel(SQLQueryError.ErrorLevel.ERROR); e.setCode(1); e.setMessage("msg"); String str = e.toString(); @@ -81,10 +81,10 @@ public void testCompareTo() { e2.setCode(2); assertTrue(e1.compareTo(e2) < 0); e2.setCode(1); - e1.setLevel("error"); - e2.setLevel("warning"); + e1.setLevel(SQLQueryError.ErrorLevel.ERROR); + e2.setLevel(SQLQueryError.ErrorLevel.WARNING); assertTrue(e1.compareTo(e2) > 0 || e1.compareTo(e2) < 0); - e2.setLevel("error"); + e2.setLevel(SQLQueryError.ErrorLevel.ERROR); e1.setMessage("a"); e2.setMessage("b"); assertTrue(e1.compareTo(e2) < 0); From f12c6e1a4b0bedc196d07a538bd926b7278cb47a Mon Sep 17 00:00:00 2001 From: HDQ Date: Wed, 16 Jul 2025 13:44:20 +0800 Subject: [PATCH 695/856] Run mvn formatter:format --- src/sqlancer/postgres/PostgresProvider.java | 6 +++--- test/sqlancer/common/query/SQLQueryErrorTest.java | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index 9fb975667..9cf4c5a75 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -279,14 +279,14 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE } Connection con = DriverManager.getConnection("jdbc:" + entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - + String dropCommand = "DROP DATABASE"; boolean forceDrop = Randomly.getBoolean(); if (forceDrop) { dropCommand += " FORCE"; } dropCommand += " IF EXISTS " + databaseName; - + globalState.getState().logStatement(dropCommand + ";"); try (Statement s = con.createStatement()) { s.execute(dropCommand); @@ -302,7 +302,7 @@ public SQLConnection createDatabase(PostgresGlobalState globalState) throws SQLE throw e; } } - + // Create database section createDatabaseCommand = getCreateDatabaseCommand(globalState); globalState.getState().logStatement(createDatabaseCommand + ";"); diff --git a/test/sqlancer/common/query/SQLQueryErrorTest.java b/test/sqlancer/common/query/SQLQueryErrorTest.java index e0065788e..dea1e7a53 100644 --- a/test/sqlancer/common/query/SQLQueryErrorTest.java +++ b/test/sqlancer/common/query/SQLQueryErrorTest.java @@ -90,4 +90,3 @@ public void testCompareTo() { assertTrue(e1.compareTo(e2) < 0); } } - From ab1bead036362dff7d986e47dacdb1e825164f79 Mon Sep 17 00:00:00 2001 From: HDQ Date: Wed, 16 Jul 2025 14:31:51 +0800 Subject: [PATCH 696/856] Rewrite hashCode method for SQLQueryError.java --- src/sqlancer/common/query/SQLQueryError.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/common/query/SQLQueryError.java b/src/sqlancer/common/query/SQLQueryError.java index 4ab253616..41f604930 100644 --- a/src/sqlancer/common/query/SQLQueryError.java +++ b/src/sqlancer/common/query/SQLQueryError.java @@ -1,5 +1,7 @@ package sqlancer.common.query; +import java.util.Objects; + public class SQLQueryError implements Comparable { public enum ErrorLevel { @@ -65,6 +67,11 @@ public boolean equals(Object that) { return false; } + @Override + public int hashCode() { + return Objects.hash(level, code, message); + } + @Override public String toString() { return String.format("Level: %s; Code: %d; Message: %s.", level, code, message); @@ -106,4 +113,4 @@ public int compareTo(SQLQueryError that) { return 0; } -} \ No newline at end of file +} From 4b0b66d6bf02e76ebe084188dbb990cfd73a796c Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sat, 26 Jul 2025 21:59:06 +0800 Subject: [PATCH 697/856] fix wrong value in the tidb_enable_stmt_summary Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index 11f599fc0..c59d0dc2f 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -47,7 +47,7 @@ private enum Action { // java.sql.SQLException: Fast analyze hasn't reached General Availability and only support analyze version 1 // currently TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), - TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions(0, 1)), + TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("", "table", "global")), TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // TIDB_ENABLE_CHUNK_RPC("tidb_enable_chunk_rpc", (r) -> Randomly.fromOptions(0, 1)); From 603d31ae8f4f16552977224e915e7cf39d09855d Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Sun, 27 Jul 2025 11:15:15 +0800 Subject: [PATCH 698/856] update tidb version as v8.5.2 Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 888c04d06..484c241d5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -609,7 +609,7 @@ jobs: - name: Set up TiDB run: | docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v7.5.1 + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" From 5aace839806814bd8e365c902f1852567aa3682f Mon Sep 17 00:00:00 2001 From: yaojiejia Date: Sun, 27 Jul 2025 18:32:32 -0400 Subject: [PATCH 699/856] Removing permision denied error --- src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java index 6c44043b8..3890d5160 100644 --- a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -13,7 +13,6 @@ public class PostgresTableSpaceGenerator { public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { this.globalState = globalState; errors.addRegexString("ERROR: (?:tablespace )?directory \".*[\\\\/]tablespace\\d+\" does not exist"); - errors.add("ERROR: permission denied"); errors.add("ERROR: already exists"); errors.add("ERROR: is not empty"); errors.add("ERROR: cannot be created because system does not support tablespaces"); From 10dc6f6726844154ad41270bb08c96e83b5c9443 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 11:56:05 +0800 Subject: [PATCH 700/856] update tidb image Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 4 ++-- .settings/org.eclipse.jdt.core.prefs | 9 ++++++--- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 484c241d5..b059b1859 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -608,8 +608,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 + docker pull hawkingrei/tidb-playground:v8.5.2 + docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:v8.5.2 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 0a659c1a7..3af1c5ad8 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.compliance=11 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,6 +28,7 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -73,6 +74,7 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -110,5 +112,6 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning +org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=1.8 +org.eclipse.jdt.core.compiler.source=11 diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index c59d0dc2f..f32e77980 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -47,7 +47,7 @@ private enum Action { // java.sql.SQLException: Fast analyze hasn't reached General Availability and only support analyze version 1 // currently TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), - TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("", "table", "global")), + TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("", "`table`", "global")), TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // TIDB_ENABLE_CHUNK_RPC("tidb_enable_chunk_rpc", (r) -> Randomly.fromOptions(0, 1)); From b02c1af5a4aadec8ac2711f4fc3baac368133611 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 12:27:14 +0800 Subject: [PATCH 701/856] update Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 4 ++-- src/sqlancer/tidb/gen/TiDBSetGenerator.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b059b1859..888c04d06 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -608,8 +608,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull hawkingrei/tidb-playground:v8.5.2 - docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:v8.5.2 + docker pull pingcap/tidb:latest + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v7.5.1 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/src/sqlancer/tidb/gen/TiDBSetGenerator.java b/src/sqlancer/tidb/gen/TiDBSetGenerator.java index f32e77980..c997f6717 100644 --- a/src/sqlancer/tidb/gen/TiDBSetGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBSetGenerator.java @@ -47,7 +47,7 @@ private enum Action { // java.sql.SQLException: Fast analyze hasn't reached General Availability and only support analyze version 1 // currently TIDB_WAIT_SPLIT_REGION_FINISH("tidb_wait_split_region_finish", (r) -> Randomly.fromOptions(0, 1)), - TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("", "`table`", "global")), + TIDB_SCATTER_REGION("global.tidb_scatter_region", (r) -> Randomly.fromOptions("``", "`table`", "global")), TIDB_ENABLE_STMT_SUMMARY("global.tidb_enable_stmt_summary", (r) -> Randomly.fromOptions(0, 1)), // TIDB_ENABLE_CHUNK_RPC("tidb_enable_chunk_rpc", (r) -> Randomly.fromOptions(0, 1)); From df514783dbc9f5dc598d4488256926a4fd3bd22e Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 14:21:03 +0800 Subject: [PATCH 702/856] update Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 2 +- .settings/org.eclipse.core.resources.prefs | 4 ++++ .settings/org.eclipse.jdt.apt.core.prefs | 4 ++++ .settings/org.eclipse.jdt.core.prefs | 9 +++------ .settings/org.eclipse.m2e.core.prefs | 4 ++++ 5 files changed, 16 insertions(+), 7 deletions(-) create mode 100644 .settings/org.eclipse.core.resources.prefs create mode 100644 .settings/org.eclipse.jdt.apt.core.prefs create mode 100644 .settings/org.eclipse.m2e.core.prefs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 888c04d06..484c241d5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -609,7 +609,7 @@ jobs: - name: Set up TiDB run: | docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v7.5.1 + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 000000000..104c8ec98 --- /dev/null +++ b/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +encoding/=UTF-8 +encoding/src=UTF-8 +encoding/test=UTF-8 diff --git a/.settings/org.eclipse.jdt.apt.core.prefs b/.settings/org.eclipse.jdt.apt.core.prefs new file mode 100644 index 000000000..dfa4f3adb --- /dev/null +++ b/.settings/org.eclipse.jdt.apt.core.prefs @@ -0,0 +1,4 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.apt.aptEnabled=true +org.eclipse.jdt.apt.genSrcDir=target/generated-sources/annotations +org.eclipse.jdt.apt.genTestSrcDir=target/generated-test-sources/test-annotations diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs index 3af1c5ad8..0a659c1a7 100644 --- a/.settings/org.eclipse.jdt.core.prefs +++ b/.settings/org.eclipse.jdt.core.prefs @@ -11,9 +11,9 @@ org.eclipse.jdt.core.compiler.annotation.nullable.secondary= org.eclipse.jdt.core.compiler.annotation.nullanalysis=disabled org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate -org.eclipse.jdt.core.compiler.codegen.targetPlatform=11 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve -org.eclipse.jdt.core.compiler.compliance=11 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate @@ -28,7 +28,6 @@ org.eclipse.jdt.core.compiler.problem.deprecationInDeprecatedCode=disabled org.eclipse.jdt.core.compiler.problem.deprecationWhenOverridingDeprecatedMethod=disabled org.eclipse.jdt.core.compiler.problem.discouragedReference=warning org.eclipse.jdt.core.compiler.problem.emptyStatement=ignore -org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error org.eclipse.jdt.core.compiler.problem.explicitlyClosedAutoCloseable=ignore org.eclipse.jdt.core.compiler.problem.fallthroughCase=ignore @@ -74,7 +73,6 @@ org.eclipse.jdt.core.compiler.problem.redundantSpecificationOfTypeArguments=igno org.eclipse.jdt.core.compiler.problem.redundantSuperinterface=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBePotentiallyStatic=ignore org.eclipse.jdt.core.compiler.problem.reportMethodCanBeStatic=ignore -org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore org.eclipse.jdt.core.compiler.problem.specialParameterHidingField=disabled org.eclipse.jdt.core.compiler.problem.staticAccessReceiver=warning org.eclipse.jdt.core.compiler.problem.suppressOptionalErrors=disabled @@ -112,6 +110,5 @@ org.eclipse.jdt.core.compiler.problem.unusedPrivateMember=warning org.eclipse.jdt.core.compiler.problem.unusedTypeParameter=ignore org.eclipse.jdt.core.compiler.problem.unusedWarningToken=warning org.eclipse.jdt.core.compiler.problem.varargsArgumentNeedCast=warning -org.eclipse.jdt.core.compiler.processAnnotations=enabled org.eclipse.jdt.core.compiler.release=disabled -org.eclipse.jdt.core.compiler.source=11 +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 000000000..f897a7f1c --- /dev/null +++ b/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 From ccf25fd730006cee0b77a2a4faa307a9e9ad9c33 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 14:23:59 +0800 Subject: [PATCH 703/856] update Signed-off-by: Weizhen Wang --- .settings/org.eclipse.core.resources.prefs | 4 ---- .settings/org.eclipse.jdt.apt.core.prefs | 4 ---- .settings/org.eclipse.m2e.core.prefs | 4 ---- 3 files changed, 12 deletions(-) delete mode 100644 .settings/org.eclipse.core.resources.prefs delete mode 100644 .settings/org.eclipse.jdt.apt.core.prefs delete mode 100644 .settings/org.eclipse.m2e.core.prefs diff --git a/.settings/org.eclipse.core.resources.prefs b/.settings/org.eclipse.core.resources.prefs deleted file mode 100644 index 104c8ec98..000000000 --- a/.settings/org.eclipse.core.resources.prefs +++ /dev/null @@ -1,4 +0,0 @@ -eclipse.preferences.version=1 -encoding/=UTF-8 -encoding/src=UTF-8 -encoding/test=UTF-8 diff --git a/.settings/org.eclipse.jdt.apt.core.prefs b/.settings/org.eclipse.jdt.apt.core.prefs deleted file mode 100644 index dfa4f3adb..000000000 --- a/.settings/org.eclipse.jdt.apt.core.prefs +++ /dev/null @@ -1,4 +0,0 @@ -eclipse.preferences.version=1 -org.eclipse.jdt.apt.aptEnabled=true -org.eclipse.jdt.apt.genSrcDir=target/generated-sources/annotations -org.eclipse.jdt.apt.genTestSrcDir=target/generated-test-sources/test-annotations diff --git a/.settings/org.eclipse.m2e.core.prefs b/.settings/org.eclipse.m2e.core.prefs deleted file mode 100644 index f897a7f1c..000000000 --- a/.settings/org.eclipse.m2e.core.prefs +++ /dev/null @@ -1,4 +0,0 @@ -activeProfiles= -eclipse.preferences.version=1 -resolveWorkspaceProjects=true -version=1 From 279049acf9715bb6f710425d87cb3d9d321f31a0 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 28 Jul 2025 14:52:02 +0800 Subject: [PATCH 704/856] all test for tidb use v8.5.2 Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 484c241d5..457ef173e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -608,7 +608,7 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:latest + docker pull pingcap/tidb:v8.5.2 docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 sleep 10 - name: Create SQLancer user @@ -635,8 +635,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:latest - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:latest + docker pull pingcap/tidb:v8.5.2 + docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" From 6c8166b53adbed337090ba8aec5e1086e3514a90 Mon Sep 17 00:00:00 2001 From: HDQ Date: Sun, 3 Aug 2025 19:34:01 +0800 Subject: [PATCH 705/856] Introduce DQE for MySQL in SQLancer and add test to workflow --- .github/workflows/main.yml | 1 + src/sqlancer/common/oracle/DQEBase.java | 156 ++++++ .../common/query/SQLQueryAdapter.java | 66 ++- .../common/query/SQLancerResultSet.java | 19 + src/sqlancer/mysql/MySQLOracleFactory.java | 7 + src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 518 ++++++++++++++++++ test/sqlancer/dbms/TestMySQLDQE.java | 21 + 7 files changed, 781 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/common/oracle/DQEBase.java create mode 100644 src/sqlancer/mysql/oracle/MySQLDQEOracle.java create mode 100644 test/sqlancer/dbms/TestMySQLDQE.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 457ef173e..7b0e4e410 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -467,6 +467,7 @@ jobs: MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLCERT + MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLDQE oceanbase: name: DBMS Tests (OceanBase) diff --git a/src/sqlancer/common/oracle/DQEBase.java b/src/sqlancer/common/oracle/DQEBase.java new file mode 100644 index 000000000..3781ea227 --- /dev/null +++ b/src/sqlancer/common/oracle/DQEBase.java @@ -0,0 +1,156 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import sqlancer.Main; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryError; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.mysql.MySQLSchema.MySQLTables; + +/* + * In DBMSs, SELECT, UPDATE and DELETE queries utilize predicates (i.e., WHERE clauses) to specify which rows to retrieve, update or delete, respectively. + * If they use the same predicate φ, they should access the same rows in a database. + * Ideally, DBMSs can adopt the same implementations for predicate evaluation in SELECT, UPDATE and DELETE queries. + * However, a DBMS usually adopts different implementations for predicate evaluation in SELECT, UPDATE and DELETE queries due to various optimization choices. + * Inconsistent implementations for predicate evaluation among these queries can cause SELECT, UPDATE and DELETE queries with the same predicate φ to access different rows. + * + * + * Inspired by this key observation, we propose Differential Query Execution(DQE), a novel and general approach to detect logic bugs in SELECT, UPDATE and DELETE queries. + * DQE solves the test oracle problem by executing SELECT, UPDATE and DELETE queries with the same predicate φ, and observing inconsistencies among their execution results. + * For example, if a row that is updated by an UPDATE query with a predicate φ does not appear in the query result of a SELECT query with the same predicate φ, a logic bug is detected in the target DBMS. + * The key challenge of DQE is to automatically obtain the accessed rows for a given SELECT, UPDATE or DELETE query. + * To address this challenge, we append two extra columns to each table in a database, to uniquely identify each row and track whether a row has been modified, respectively. + * We further rewrite SELECT and UPDATE queries to identify their accessed rows. + * + * more information see [DQE paper](https://ieeexplore.ieee.org/document/10172736) + */ + +public abstract class DQEBase> { + + public static final String COLUMN_ROWID = "rowId"; + public static final String COLUMN_UPDATED = "updated"; + + protected final S state; + protected final ExpectedErrors selectExpectedErrors = new ExpectedErrors(); + protected final ExpectedErrors updateExpectedErrors = new ExpectedErrors(); + protected final ExpectedErrors deleteExpectedErrors = new ExpectedErrors(); + + protected final Main.StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + + public DQEBase(S state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } + + public abstract String generateSelectStatement(MySQLTables tables, String tableName, String whereClauseStr); + + public abstract String generateUpdateStatement(MySQLTables tables, String tableName, String whereClauseStr); + + public abstract String generateDeleteStatement(MySQLTables tables, String tableName, String whereClauseStr); + + /** + * Add auxiliary columns to the database A abstract method, subclasses need to implement it. + * + * @param table + * + * @throws SQLException + */ + public abstract void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException; + + public void dropAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { + String tableName = table.getName(); + String dropColumnRowId = String.format("ALTER TABLE %s DROP COLUMN %s", tableName, COLUMN_ROWID); + new SQLQueryAdapter(dropColumnRowId).execute(state); + String dropColumnUpdated = String.format("ALTER TABLE %s DROP COLUMN %s", tableName, COLUMN_UPDATED); + new SQLQueryAdapter(dropColumnUpdated).execute(state); + } + + // This interface is to record Error code + public interface UpdateErrorCodes { + + } + + public interface ErrorCodeStrategy { + Set getUpdateSpecificErrorCodes(); + + Set getDeleteSpecificErrorCodes(); + + } + + /** + * The core idea of DQE is that the SELECT, UPDATE and DELETE queries with the same predicate φ should access the + * same rows. If these queries access different rows, DQE reveals a potential logic bug in the target DBMS. + */ + public static class SQLQueryResult { + + private final Map, Set> accessedRows; // Table name with respect rows + private final List queryErrors; + + public SQLQueryResult(Map, Set> accessedRows, + List queryErrors) { + this.accessedRows = accessedRows; + this.queryErrors = queryErrors; + } + + public Map, Set> getAccessedRows() { + return accessedRows; + } + + public List getQueryErrors() { + return queryErrors; + } + + public boolean hasEmptyErrors() { + return queryErrors.isEmpty(); + } + + public boolean hasErrors() { + return !hasEmptyErrors(); + } + + public boolean hasSameErrors(SQLQueryResult that) { + if (queryErrors.size() != that.getQueryErrors().size()) { + return false; + } else { + Collections.sort(queryErrors); + Collections.sort(that.getQueryErrors()); + for (int i = 0; i < queryErrors.size(); i++) { + if (!queryErrors.get(i).equals(that.getQueryErrors().get(i))) { + return false; + } + } + } + return true; + } + + public boolean hasAccessedRows() { + if (accessedRows.isEmpty()) { + return false; + } + for (Set accessedRow : accessedRows.values()) { + if (!accessedRow.isEmpty()) { + return true; + } + } + return false; + } + + public boolean hasSameAccessedRows(SQLQueryResult that) { + return accessedRows.equals(that.getAccessedRows()); + } + + } +} diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 3784ed635..0b94ae3fb 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -81,17 +81,55 @@ public String getUnterminatedQueryString() { return result; } + /** + * This method is used to mostly oracles, which need to report exceptions. We set the reportException parameter to + * true by default meaning that exceptions are reported. + * + * @param globalState + * @param fills + * + * @return whether the query was executed successfully + * + * @param + * + * @throws SQLException + */ @Override public > boolean execute(G globalState, String... fills) throws SQLException { + return execute(globalState, true, fills); + } + + /** + * This method is used to DQE oracles, DQE does not check exception separately, while other testing methods may + * need. We use reportException to control this behavior. For a specific DBMS used DQE oracle, we call this method + * and pass a boolean value of false as an argument. + * + * @param globalState + * @param reportException + * @param fills + * + * @return whether the query was executed successfully + * + * @param + * + * @throws SQLException + */ + public > boolean execute(G globalState, boolean reportException, + String... fills) throws SQLException { + return internalExecute(globalState.getConnection(), reportException, fills); + } + + protected > boolean internalExecute(SQLConnection connection, + boolean reportException, String... fills) throws SQLException { Statement s; if (fills.length > 0) { - s = globalState.getConnection().prepareStatement(fills[0]); + s = connection.prepareStatement(fills[0]); for (int i = 1; i < fills.length; i++) { ((PreparedStatement) s).setString(i, fills[i]); } } else { - s = globalState.getConnection().createStatement(); + s = connection.createStatement(); } try { if (fills.length > 0) { @@ -103,7 +141,9 @@ public String getUnterminatedQueryString() { return true; } catch (Exception e) { Main.nrUnsuccessfulActions.addAndGet(1); - checkException(e); + if (reportException) { + checkException(e); + } return false; } finally { s.close(); @@ -127,14 +167,24 @@ public void checkException(Exception e) throws AssertionError { @Override public > SQLancerResultSet executeAndGet(G globalState, String... fills) throws SQLException { + return executeAndGet(globalState, true, fills); + } + + public > SQLancerResultSet executeAndGet(G globalState, + boolean reportException, String... fills) throws SQLException { + return internalExecuteAndGet(globalState.getConnection(), reportException, fills); + } + + protected > SQLancerResultSet internalExecuteAndGet( + SQLConnection connection, boolean reportException, String... fills) throws SQLException { Statement s; if (fills.length > 0) { - s = globalState.getConnection().prepareStatement(fills[0]); + s = connection.prepareStatement(fills[0]); for (int i = 1; i < fills.length; i++) { ((PreparedStatement) s).setString(i, fills[i]); } } else { - s = globalState.getConnection().createStatement(); + s = connection.createStatement(); } ResultSet result; try { @@ -151,9 +201,11 @@ public void checkException(Exception e) throws AssertionError { } catch (Exception e) { s.close(); Main.nrUnsuccessfulActions.addAndGet(1); - checkException(e); + if (reportException) { + checkException(e); + } + return null; } - return null; } @Override diff --git a/src/sqlancer/common/query/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java index d1221a7f7..9466bcad2 100644 --- a/src/sqlancer/common/query/SQLancerResultSet.java +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -4,6 +4,8 @@ import java.sql.ResultSet; import java.sql.SQLException; +import sqlancer.common.query.SQLQueryError.ErrorLevel; + public class SQLancerResultSet implements Closeable { ResultSet rs; @@ -42,6 +44,23 @@ public String getString(int i) throws SQLException { } } + // This method is to fetch the result of SQL: "SHOW WARNINGS;" + public ErrorLevel getErrorLevel(String colName) throws SQLException { + String res = rs.getString(colName); + if (res == null) { + return ErrorLevel.WARNING; + } + return "ERROR".equalsIgnoreCase(res.trim()) ? ErrorLevel.ERROR : ErrorLevel.WARNING; + } + + public String getString(String colName) throws SQLException { + return rs.getString(colName); + } + + public int getInt(String colName) throws SQLException { + return rs.getInt(colName); + } + public boolean isClosed() throws SQLException { return rs.isClosed(); } diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index a0170d685..83e08677a 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -10,6 +10,7 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLancerResultSet; import sqlancer.mysql.gen.MySQLExpressionGenerator; +import sqlancer.mysql.oracle.MySQLDQEOracle; import sqlancer.mysql.oracle.MySQLDQPOracle; import sqlancer.mysql.oracle.MySQLFuzzer; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; @@ -75,5 +76,11 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLDQPOracle(globalState); } + }, + DQE { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLDQEOracle(globalState); + } }; } diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java new file mode 100644 index 000000000..142e56c22 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -0,0 +1,518 @@ +package sqlancer.mysql.oracle; + +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import com.beust.jcommander.Strings; + +import sqlancer.Randomly; +import sqlancer.common.oracle.DQEBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryError; +import sqlancer.common.query.SQLancerResultSet; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractTable; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.gen.MySQLExpressionGenerator; + +public class MySQLDQEOracle extends DQEBase implements TestOracle { + + private final MySQLSchema schema; + private static final String APPEND_ORDER_BY = "%s ORDER BY %s"; + private static final String APPEND_LIMIT = "%s LIMIT %d"; + private final List orderColumns = new ArrayList<>(); + private boolean generateLimit; + private boolean generateOrderBy; + private boolean operateOnSingleTable; + private int limit; + + public MySQLDQEOracle(MySQLGlobalState state) { + super(state); + schema = state.getSchema(); + + MySQLErrors.addExpressionErrors(selectExpectedErrors); + + MySQLErrors.addExpressionErrors(updateExpectedErrors); + updateExpectedErrors.add("cannot be null"); + updateExpectedErrors.add("Duplicate entry"); + updateExpectedErrors.add("The value specified for generated column"); + + MySQLErrors.addExpressionErrors(deleteExpectedErrors); + deleteExpectedErrors.add("a foreign key constraint fails"); + } + + @Override + public String generateSelectStatement(MySQLTables tables, String tableName, String whereClauseStr) { + operateOnSingleTable = tables.getTables().size() == 1; + List selectColumns = new ArrayList<>(); + for (MySQLTable table : tables.getTables()) { + selectColumns.add(table.getName() + "." + COLUMN_ROWID); + } + if (operateOnSingleTable && Randomly.getBooleanWithSmallProbability()) { + generateOrderBy = true; + // generate order by columns + for (MySQLColumn column : Randomly.nonEmptySubset(tables.getColumns())) { + orderColumns.add(column.getFullQualifiedName()); + } + + if (Randomly.getBooleanWithRatherLowProbability()) { + generateLimit = true; + limit = (int) Randomly.getNotCachedInteger(1, 10); + } + } + + String selectStmt = String.format("SELECT %s FROM %s WHERE %s", Strings.join(",", selectColumns).toLowerCase(), + tableName, whereClauseStr); + if (generateOrderBy) { + selectStmt = String.format(APPEND_ORDER_BY, selectStmt, String.join(",", orderColumns)); + if (generateLimit) { + selectStmt = String.format(APPEND_LIMIT, selectStmt, limit); + } + } + return selectStmt; + } + + @Override + public String generateUpdateStatement(MySQLTables tables, String tableName, String whereClauseStr) { + List updateColumns = new ArrayList<>(); + for (MySQLTable table : tables.getTables()) { + updateColumns.add(String.format("%s = 1", table.getName() + "." + COLUMN_UPDATED)); + } + String updateStmt = String.format("UPDATE %s SET %s WHERE %s", tableName, Strings.join(",", updateColumns), + whereClauseStr); + if (generateOrderBy) { + updateStmt = String.format(APPEND_ORDER_BY, updateStmt, String.join(",", orderColumns)); + if (generateLimit) { + updateStmt = String.format(APPEND_LIMIT, updateStmt, limit); + } + } + return updateStmt; + } + + @Override + public String generateDeleteStatement(MySQLTables tables, String tableName, String whereClauseStr) { + String deleteStmt; + if (operateOnSingleTable) { + deleteStmt = String.format("DELETE FROM %s WHERE %s", tableName, whereClauseStr); + if (generateOrderBy) { + deleteStmt = String.format(APPEND_ORDER_BY, deleteStmt, String.join(",", orderColumns)); + if (generateLimit) { + deleteStmt = String.format(APPEND_LIMIT, deleteStmt, limit); + } + } + } else { + deleteStmt = String.format("DELETE %s FROM %s WHERE %s", tableName, tableName, whereClauseStr); + } + return deleteStmt; + } + + @Override + public void check() throws SQLException { + + MySQLTables tables = schema.getRandomTableNonEmptyTables(); + String tableName = tables.getTables().stream().map(AbstractTable::getName).collect(Collectors.joining(",")); + + // DQE does not support aggregate functions, windows functions + // This method does not generate them, may need some configurations if they can be generated + MySQLExpressionGenerator expressionGenerator = new MySQLExpressionGenerator(state) + .setColumns(tables.getColumns()); + MySQLExpression whereClause = expressionGenerator.generateExpression(); + + // MySQLVisitor is not deterministic, we should keep it only once. + // Especially, in MySQLUnaryPostfixOperation and MySQLUnaryPrefixOperation + String whereClauseStr = MySQLVisitor.asString(whereClause); + + // Generate a SELECT statement + String selectStmt = generateSelectStatement(tables, tableName, whereClauseStr); + + // Generate an UPDATE statement + String updateStmt = generateUpdateStatement(tables, tableName, whereClauseStr); + + // Generate a DELETE statement + String deleteStmt = generateDeleteStatement(tables, tableName, whereClauseStr); + + for (MySQLTable table : tables.getTables()) { + addAuxiliaryColumns(table); + } + + state.getState().getLocalState().log(selectStmt); + SQLQueryResult selectExecutionResult = executeSelect(selectStmt, tables); + state.getState().getLocalState().log(selectExecutionResult.getAccessedRows().values().toString()); + state.getState().getLocalState().log(selectExecutionResult.getQueryErrors().toString()); + + state.getState().getLocalState().log(updateStmt); + SQLQueryResult updateExecutionResult = executeUpdate(updateStmt, tables); + state.getState().getLocalState().log(updateExecutionResult.getAccessedRows().values().toString()); + state.getState().getLocalState().log(updateExecutionResult.getQueryErrors().toString()); + + state.getState().getLocalState().log(deleteStmt); + SQLQueryResult deleteExecutionResult = executeDelete(deleteStmt, tables); + state.getState().getLocalState().log(deleteExecutionResult.getAccessedRows().values().toString()); + state.getState().getLocalState().log(deleteExecutionResult.getQueryErrors().toString()); + + String compareSelectAndUpdate = compareSelectAndUpdate(selectExecutionResult, updateExecutionResult); + String compareSelectAndDelete = compareSelectAndDelete(selectExecutionResult, deleteExecutionResult); + String compareUpdateAndDelete = compareUpdateAndDelete(updateExecutionResult, deleteExecutionResult); + + String errorMessage = compareSelectAndUpdate == null ? "" : compareSelectAndUpdate + "\n"; + errorMessage += compareSelectAndDelete == null ? "" : compareSelectAndDelete + "\n"; + errorMessage += compareUpdateAndDelete == null ? "" : compareUpdateAndDelete + "\n"; + + if (!errorMessage.equals("")) { + throw new AssertionError(errorMessage); + } else { + state.getState().getLocalState().log("PASS"); + } + + for (MySQLTable table : tables.getTables()) { + dropAuxiliaryColumns(table); + } + } + + public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult updateResult) { + if (updateResult.hasEmptyErrors()) { + if (selectResult.hasErrors()) { + return "SELECT has errors, but UPDATE does not."; + } + if (!selectResult.hasSameAccessedRows(updateResult)) { + return "SELECT accessed different rows from UPDATE."; + } + return null; + } else { // update has errors + if (hasUpdateSpecificErrors(updateResult)) { + if (updateResult.hasAccessedRows()) { + return "UPDATE accessed non-empty rows when specific errors happen."; + } else { + // we do not compare update with select when update has specific errors + return null; + } + } + + // update errors should all appear in the select errors + List queryErrors = new ArrayList<>(selectResult.getQueryErrors()); + for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { + SQLQueryError updateError = updateResult.getQueryErrors().get(i); + boolean found = false; + for (int j = 0; j < queryErrors.size(); j++) { + SQLQueryError selectError = queryErrors.get(j); + if (selectError.hasSameCodeAndMessage(updateError)) { + queryErrors.remove(selectError); + found = true; + break; + } + } + if (!found) { + return "SELECT has different errors from UPDATE."; + } + } + + if (hasStopErrors(updateResult)) { + if (updateResult.hasAccessedRows()) { + return "UPDATE accessed non-empty rows when stop errors happen."; + } + } else { + if (!selectResult.hasSameAccessedRows(updateResult)) { + return "SELECT accessed different rows from UPDATE when errors happen."; + } + + } + + return null; + } + } + + public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { + if (deleteResult.hasEmptyErrors()) { + if (selectResult.hasErrors()) { + return "SELECT has errors, but DELETE does not."; + } + if (!selectResult.hasSameAccessedRows(deleteResult)) { + return "SELECT accessed different rows from DELETE."; + } + return null; + } else { // delete has errors + if (hasDeleteSpecificErrors(deleteResult)) { + if (deleteResult.hasAccessedRows()) { + return "DELETE accessed non-empty rows when specific errors happen."; + } else { + // we do not compare delete with select when delete has specific errors + return null; + } + } + + // delete errors should all appear in the select errors + List queryErrors = new ArrayList<>(selectResult.getQueryErrors()); + for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { + SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); + boolean found = false; + for (int j = 0; j < queryErrors.size(); j++) { + SQLQueryError selectError = queryErrors.get(j); + if (selectError.hasSameCodeAndMessage(deleteError)) { + queryErrors.remove(deleteError); + found = true; + break; + } + } + if (!found) { + return "SELECT has different errors from DELETE."; + } + } + + if (hasStopErrors(deleteResult)) { + if (deleteResult.hasAccessedRows()) { + return "DELETE accessed non-empty rows when stop errors happen."; + } + } else { + if (!selectResult.hasSameAccessedRows(deleteResult)) { + return "SELECT accessed different rows from DELETE when errors happen."; + } + } + + return null; + } + } + + public String compareUpdateAndDelete(SQLQueryResult updateResult, SQLQueryResult deleteResult) { + if (updateResult.hasEmptyErrors() && deleteResult.hasEmptyErrors()) { + if (updateResult.hasSameAccessedRows(deleteResult)) { + return null; + } else { + return "UPDATE accessed different rows from DELETE."; + } + } else { // update or delete has errors + boolean hasSpecificErrors = false; + + if (hasUpdateSpecificErrors(updateResult)) { + hasSpecificErrors = true; + if (updateResult.hasAccessedRows()) { + return "UPDATE accessed non-empty rows when specific errors happen."; + } + } + + if (hasDeleteSpecificErrors(deleteResult)) { + hasSpecificErrors = true; + if (deleteResult.hasAccessedRows()) { + return "DELETE accessed non-empty rows when specific errors happen."; + } + } + + // when one of these statements has specific errors, do not compare them + if (hasSpecificErrors) { + return null; + } + + if (!updateResult.hasSameErrors(deleteResult)) { + return "UPDATE has different errors from DELETE."; + } else { + if (!hasStopErrors(updateResult)) { + if (!updateResult.hasSameAccessedRows(deleteResult)) { + return "UPDATE accessed different rows from DELETE."; + } + } else { + if (updateResult.hasAccessedRows() || deleteResult.hasAccessedRows()) { + return "UPDATE or DELETE accessed non-empty rows when stop errors happen."; + } + } + } + + return null; + } + } + + /* + * when update violates column constraints, such as not null, unique, primary key and generated column, we cannot + * compare it with other queries. + */ + private boolean hasUpdateSpecificErrors(SQLQueryResult updateResult) { + return updateResult.getQueryErrors().stream().anyMatch( + error -> new MySQLErrorCodeStrategy().getUpdateSpecificErrorCodes().contains(error.getCode())); + } + + /* + * when delete violates column constraints, such as foreign key, we cannot compare it with other queries. + */ + private boolean hasDeleteSpecificErrors(SQLQueryResult deleteResult) { + return deleteResult.getQueryErrors().stream().anyMatch( + error -> new MySQLErrorCodeStrategy().getDeleteSpecificErrorCodes().contains(error.getCode())); + + } + + private boolean hasStopErrors(SQLQueryResult queryResult) { + return queryResult.getQueryErrors().stream() + .anyMatch(error -> error.getLevel() == SQLQueryError.ErrorLevel.ERROR); + } + + private SQLQueryResult executeSelect(String selectStmt, MySQLTables tables) throws SQLException { + Map, Set> accessedRows = new HashMap<>(); + List queryErrors; + SQLancerResultSet resultSet = null; + try { + resultSet = new SQLQueryAdapter(selectStmt, selectExpectedErrors).executeAndGet(state, false); + } catch (SQLException ignored) { + // we ignore this error, and use get errors to catch it + } finally { + queryErrors = getErrors(); + + if (resultSet != null) { + for (MySQLTable table : tables.getTables()) { + HashSet rows = new HashSet<>(); + accessedRows.put(table, rows); + } + while (resultSet.next()) { + for (MySQLTable table : tables.getTables()) { + accessedRows.get(table).add(resultSet.getString(table.getName() + "." + COLUMN_ROWID)); + } + } + resultSet.close(); + } + } + + return new SQLQueryResult(accessedRows, queryErrors); + } + + private SQLQueryResult executeUpdate(String updateStmt, MySQLTables tables) throws SQLException { + Map, Set> accessedRows = new HashMap<>(); + List queryErrors; + try { + new SQLQueryAdapter("BEGIN").execute(state, false); + new SQLQueryAdapter(updateStmt, updateExpectedErrors).execute(state, false); + } catch (SQLException ignored) { + // we ignore this error, and we use get errors to catch it + } finally { + queryErrors = getErrors(); + + for (MySQLTable table : tables.getTables()) { + String tableName = table.getName(); + String rowId = tableName + "." + COLUMN_ROWID; + String updated = tableName + "." + COLUMN_UPDATED; + String selectRowIdWithUpdated = String.format("SELECT %s FROM %s WHERE %s = 1", rowId, tableName, + updated); + SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowIdWithUpdated).executeAndGet(state, false); + HashSet rows = new HashSet<>(); + if (resultSet != null) { + while (resultSet.next()) { + rows.add(resultSet.getString(rowId)); + } + resultSet.close(); + } + accessedRows.put(table, rows); + } + + new SQLQueryAdapter("ROLLBACK").execute(state, false); + } + + return new SQLQueryResult(accessedRows, queryErrors); + } + + private SQLQueryResult executeDelete(String deleteStmt, MySQLTables tables) throws SQLException { + Map, Set> accessedRows = new HashMap<>(); + List queryErrors; + try { + for (MySQLTable table : tables.getTables()) { + String tableName = table.getName(); + String rowId = tableName + "." + COLUMN_ROWID; + String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); + SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowId).executeAndGet(state, false); + HashSet rows = new HashSet<>(); + if (resultSet != null) { + while (resultSet.next()) { + rows.add(resultSet.getString(rowId)); + } + resultSet.close(); + } + accessedRows.put(table, rows); + } + + new SQLQueryAdapter("BEGIN").execute(state, false); + new SQLQueryAdapter(deleteStmt, deleteExpectedErrors).execute(state, false); + } catch (SQLException ignored) { + // we ignore this error, and use get errors to catch it + } finally { + queryErrors = getErrors(); + + for (MySQLTable table : tables.getTables()) { + String tableName = table.getName(); + String rowId = tableName + "." + COLUMN_ROWID; + String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); + SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowId).executeAndGet(state, false); + HashSet rows = new HashSet<>(); + if (resultSet != null) { + while (resultSet.next()) { + rows.add(resultSet.getString(rowId)); + } + resultSet.close(); + } + + accessedRows.get(table).removeAll(rows); + } + + new SQLQueryAdapter("ROLLBACK").execute(state, false); + } + + return new SQLQueryResult(accessedRows, queryErrors); + } + + private List getErrors() throws SQLException { + SQLancerResultSet resultSet = new SQLQueryAdapter("SHOW WARNINGS").executeAndGet(state, false); + List queryErrors = new ArrayList<>(); + if (resultSet != null) { + while (resultSet.next()) { + SQLQueryError queryError = new SQLQueryError(); + queryError.setLevel(resultSet.getErrorLevel("Level")); + queryError.setCode(resultSet.getInt("Code")); + queryError.setMessage(resultSet.getString("Message")); + queryErrors.add(queryError); + } + resultSet.close(); + } + + return queryErrors; + } + + @Override + public void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { + String tableName = table.getName(); + + String addColumnRowID = String.format("ALTER TABLE %s ADD %s TEXT", tableName, COLUMN_ROWID); + new SQLQueryAdapter(addColumnRowID).execute(state, false); + state.getState().getLocalState().log(addColumnRowID); + + String addColumnUpdated = String.format("ALTER TABLE %s ADD %s INT DEFAULT 0", tableName, COLUMN_UPDATED); + new SQLQueryAdapter(addColumnUpdated).execute(state, false); + state.getState().getLocalState().log(addColumnUpdated); + + String updateRowsWithUniqueID = String.format("UPDATE %s SET %s = UUID()", tableName, COLUMN_ROWID); + new SQLQueryAdapter(updateRowsWithUniqueID).execute(state, false); + state.getState().getLocalState().log(updateRowsWithUniqueID); + } + + public static class MySQLErrorCodeStrategy implements ErrorCodeStrategy { + @Override + public Set getUpdateSpecificErrorCodes() { + // 1048, Column 'c0' cannot be null + // 1062, Duplicate entry '2' for key 't1.i0 + // 3105, The value specified for generated column 'c1' in table 't1' is not allowed + return Set.of(1048, 1062, 3105); + } + + @Override + public Set getDeleteSpecificErrorCodes() { + // 1451, Cannot delete or update a parent row: a foreign key constraint fails + return Set.of(1451); + } + } +} diff --git a/test/sqlancer/dbms/TestMySQLDQE.java b/test/sqlancer/dbms/TestMySQLDQE.java new file mode 100644 index 000000000..c137c1d0a --- /dev/null +++ b/test/sqlancer/dbms/TestMySQLDQE.java @@ -0,0 +1,21 @@ +package sqlancer.dbms; + +import org.junit.jupiter.api.Test; +import sqlancer.Main; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +public class TestMySQLDQE { + + @Test + public void testMySQL() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.MYSQL_ENV)); + // Run with 0 queries as there are false positives for every mutation + assertEquals(0, + Main.executeMain(new String[] { "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, + "--max-expression-depth", "1", "--num-threads", "4", "--num-queries", TestConfig.NUM_QUERIES, + "mysql", "--oracle", "DQE" })); + } + +} From 1f64426f4cf3460ee3bd984e44382265a1059b06 Mon Sep 17 00:00:00 2001 From: HDQ Date: Fri, 8 Aug 2025 14:01:29 +0800 Subject: [PATCH 706/856] Fix: remove MySQL specific logic and reuse existing method --- .../common/query/SQLancerResultSet.java | 11 ------ src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 39 +++++-------------- 2 files changed, 10 insertions(+), 40 deletions(-) diff --git a/src/sqlancer/common/query/SQLancerResultSet.java b/src/sqlancer/common/query/SQLancerResultSet.java index 9466bcad2..912a6a882 100644 --- a/src/sqlancer/common/query/SQLancerResultSet.java +++ b/src/sqlancer/common/query/SQLancerResultSet.java @@ -4,8 +4,6 @@ import java.sql.ResultSet; import java.sql.SQLException; -import sqlancer.common.query.SQLQueryError.ErrorLevel; - public class SQLancerResultSet implements Closeable { ResultSet rs; @@ -44,15 +42,6 @@ public String getString(int i) throws SQLException { } } - // This method is to fetch the result of SQL: "SHOW WARNINGS;" - public ErrorLevel getErrorLevel(String colName) throws SQLException { - String res = rs.getString(colName); - if (res == null) { - return ErrorLevel.WARNING; - } - return "ERROR".equalsIgnoreCase(res.trim()) ? ErrorLevel.ERROR : ErrorLevel.WARNING; - } - public String getString(String colName) throws SQLException { return rs.getString(colName); } diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index 142e56c22..d4e15de01 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -1,5 +1,7 @@ package sqlancer.mysql.oracle; +import static sqlancer.ComparatorHelper.getResultSetFirstColumnAsString; + import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; @@ -136,13 +138,10 @@ public void check() throws SQLException { // Especially, in MySQLUnaryPostfixOperation and MySQLUnaryPrefixOperation String whereClauseStr = MySQLVisitor.asString(whereClause); - // Generate a SELECT statement String selectStmt = generateSelectStatement(tables, tableName, whereClauseStr); - // Generate an UPDATE statement String updateStmt = generateUpdateStatement(tables, tableName, whereClauseStr); - // Generate a DELETE statement String deleteStmt = generateDeleteStatement(tables, tableName, whereClauseStr); for (MySQLTable table : tables.getTables()) { @@ -401,14 +400,8 @@ private SQLQueryResult executeUpdate(String updateStmt, MySQLTables tables) thro String updated = tableName + "." + COLUMN_UPDATED; String selectRowIdWithUpdated = String.format("SELECT %s FROM %s WHERE %s = 1", rowId, tableName, updated); - SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowIdWithUpdated).executeAndGet(state, false); - HashSet rows = new HashSet<>(); - if (resultSet != null) { - while (resultSet.next()) { - rows.add(resultSet.getString(rowId)); - } - resultSet.close(); - } + HashSet rows = new HashSet<>( + getResultSetFirstColumnAsString(selectRowIdWithUpdated, updateExpectedErrors, state)); accessedRows.put(table, rows); } @@ -426,14 +419,8 @@ private SQLQueryResult executeDelete(String deleteStmt, MySQLTables tables) thro String tableName = table.getName(); String rowId = tableName + "." + COLUMN_ROWID; String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); - SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowId).executeAndGet(state, false); - HashSet rows = new HashSet<>(); - if (resultSet != null) { - while (resultSet.next()) { - rows.add(resultSet.getString(rowId)); - } - resultSet.close(); - } + HashSet rows = new HashSet<>( + getResultSetFirstColumnAsString(selectRowId, deleteExpectedErrors, state)); accessedRows.put(table, rows); } @@ -448,15 +435,8 @@ private SQLQueryResult executeDelete(String deleteStmt, MySQLTables tables) thro String tableName = table.getName(); String rowId = tableName + "." + COLUMN_ROWID; String selectRowId = String.format("SELECT %s FROM %s", rowId, tableName); - SQLancerResultSet resultSet = new SQLQueryAdapter(selectRowId).executeAndGet(state, false); - HashSet rows = new HashSet<>(); - if (resultSet != null) { - while (resultSet.next()) { - rows.add(resultSet.getString(rowId)); - } - resultSet.close(); - } - + HashSet rows = new HashSet<>( + getResultSetFirstColumnAsString(selectRowId, deleteExpectedErrors, state)); accessedRows.get(table).removeAll(rows); } @@ -472,7 +452,8 @@ private List getErrors() throws SQLException { if (resultSet != null) { while (resultSet.next()) { SQLQueryError queryError = new SQLQueryError(); - queryError.setLevel(resultSet.getErrorLevel("Level")); + queryError.setLevel(resultSet.getString("Level").equalsIgnoreCase("ERROR") + ? SQLQueryError.ErrorLevel.ERROR : SQLQueryError.ErrorLevel.WARNING); queryError.setCode(resultSet.getInt("Code")); queryError.setMessage(resultSet.getString("Message")); queryErrors.add(queryError); From 83cf8995f467467fc611aef71b1f66d1127c69de Mon Sep 17 00:00:00 2001 From: HDQ Date: Fri, 8 Aug 2025 16:13:48 +0800 Subject: [PATCH 707/856] refactor(MySQLDQEOracle): extract duplicated code as method --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 54 +++++++++---------- 1 file changed, 25 insertions(+), 29 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index d4e15de01..0e43d7a86 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -171,7 +171,7 @@ public void check() throws SQLException { errorMessage += compareSelectAndDelete == null ? "" : compareSelectAndDelete + "\n"; errorMessage += compareUpdateAndDelete == null ? "" : compareUpdateAndDelete + "\n"; - if (!errorMessage.equals("")) { + if (!errorMessage.isEmpty()) { throw new AssertionError(errorMessage); } else { state.getState().getLocalState().log("PASS"); @@ -190,7 +190,6 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult if (!selectResult.hasSameAccessedRows(updateResult)) { return "SELECT accessed different rows from UPDATE."; } - return null; } else { // update has errors if (hasUpdateSpecificErrors(updateResult)) { if (updateResult.hasAccessedRows()) { @@ -202,19 +201,10 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult } // update errors should all appear in the select errors - List queryErrors = new ArrayList<>(selectResult.getQueryErrors()); + List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); - boolean found = false; - for (int j = 0; j < queryErrors.size(); j++) { - SQLQueryError selectError = queryErrors.get(j); - if (selectError.hasSameCodeAndMessage(updateError)) { - queryErrors.remove(selectError); - found = true; - break; - } - } - if (!found) { + if (!isFound(selectErrors, updateError)) { return "SELECT has different errors from UPDATE."; } } @@ -227,11 +217,28 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult if (!selectResult.hasSameAccessedRows(updateResult)) { return "SELECT accessed different rows from UPDATE when errors happen."; } - } + } + return null; + } - return null; + /** + * + * @param selectErrors selectQueryErrors + * @param targetError update or delete queryError + * @return is targetError found in selectQueryErrors + */ + private static boolean isFound(List selectErrors, SQLQueryError targetError) { + boolean found = false; + for (int i = 0; i < selectErrors.size(); i++) { + SQLQueryError selectError = selectErrors.get(i); + if (selectError.hasSameCodeAndMessage(targetError)) { + selectErrors.remove(i); + found = true; + break; + } } + return found; } public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { @@ -242,7 +249,6 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult if (!selectResult.hasSameAccessedRows(deleteResult)) { return "SELECT accessed different rows from DELETE."; } - return null; } else { // delete has errors if (hasDeleteSpecificErrors(deleteResult)) { if (deleteResult.hasAccessedRows()) { @@ -254,19 +260,10 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult } // delete errors should all appear in the select errors - List queryErrors = new ArrayList<>(selectResult.getQueryErrors()); + List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); - boolean found = false; - for (int j = 0; j < queryErrors.size(); j++) { - SQLQueryError selectError = queryErrors.get(j); - if (selectError.hasSameCodeAndMessage(deleteError)) { - queryErrors.remove(deleteError); - found = true; - break; - } - } - if (!found) { + if (!isFound(selectErrors, deleteError)) { return "SELECT has different errors from DELETE."; } } @@ -280,9 +277,8 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult return "SELECT accessed different rows from DELETE when errors happen."; } } - - return null; } + return null; } public String compareUpdateAndDelete(SQLQueryResult updateResult, SQLQueryResult deleteResult) { From e1b5861557b01d4edbcbf766b0c9177d5e1c4f93 Mon Sep 17 00:00:00 2001 From: luliqwerty Date: Fri, 8 Aug 2025 16:29:31 +0800 Subject: [PATCH 708/856] refactor(MySQLDQEOracle): run mvn formatter:format --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index 0e43d7a86..bb6737215 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -224,8 +224,11 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult /** * - * @param selectErrors selectQueryErrors - * @param targetError update or delete queryError + * @param selectErrors + * selectQueryErrors + * @param targetError + * update or delete queryError + * * @return is targetError found in selectQueryErrors */ private static boolean isFound(List selectErrors, SQLQueryError targetError) { From 131febb402173a0477840e68de28e46a75c5bc77 Mon Sep 17 00:00:00 2001 From: luliqwerty Date: Sun, 10 Aug 2025 22:08:46 +0800 Subject: [PATCH 709/856] refactor: fix some code style - DQEBase: remove unnecessary method - MySQLDQEOracle: simplify adding expectedErrors - MySQLErrors: add a expectedErrors --- src/sqlancer/common/oracle/DQEBase.java | 7 ------- src/sqlancer/mysql/MySQLErrors.java | 1 + src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 13 +++++-------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/common/oracle/DQEBase.java b/src/sqlancer/common/oracle/DQEBase.java index 3781ea227..ba9af3bd1 100644 --- a/src/sqlancer/common/oracle/DQEBase.java +++ b/src/sqlancer/common/oracle/DQEBase.java @@ -1,7 +1,6 @@ package sqlancer.common.oracle; import java.sql.SQLException; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -117,16 +116,10 @@ public boolean hasEmptyErrors() { return queryErrors.isEmpty(); } - public boolean hasErrors() { - return !hasEmptyErrors(); - } - public boolean hasSameErrors(SQLQueryResult that) { if (queryErrors.size() != that.getQueryErrors().size()) { return false; } else { - Collections.sort(queryErrors); - Collections.sort(that.getQueryErrors()); for (int i = 0; i < queryErrors.size(); i++) { if (!queryErrors.get(i).equals(that.getQueryErrors().get(i))) { return false; diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 13159e49e..f01b30e03 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -53,6 +53,7 @@ public static List getInsertUpdateErrors() { errors.add("Data truncated for functional index"); errors.add("cannot be null"); errors.add("Incorrect decimal value"); + errors.add("The value specified for generated column"); return errors; } diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index bb6737215..4585fc830 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -4,6 +4,7 @@ import java.sql.SQLException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -49,9 +50,7 @@ public MySQLDQEOracle(MySQLGlobalState state) { MySQLErrors.addExpressionErrors(selectExpectedErrors); MySQLErrors.addExpressionErrors(updateExpectedErrors); - updateExpectedErrors.add("cannot be null"); - updateExpectedErrors.add("Duplicate entry"); - updateExpectedErrors.add("The value specified for generated column"); + MySQLErrors.addInsertUpdateErrors(updateExpectedErrors); MySQLErrors.addExpressionErrors(deleteExpectedErrors); deleteExpectedErrors.add("a foreign key constraint fails"); @@ -173,8 +172,6 @@ public void check() throws SQLException { if (!errorMessage.isEmpty()) { throw new AssertionError(errorMessage); - } else { - state.getState().getLocalState().log("PASS"); } for (MySQLTable table : tables.getTables()) { @@ -184,7 +181,7 @@ public void check() throws SQLException { public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult updateResult) { if (updateResult.hasEmptyErrors()) { - if (selectResult.hasErrors()) { + if (!selectResult.hasEmptyErrors()) { return "SELECT has errors, but UPDATE does not."; } if (!selectResult.hasSameAccessedRows(updateResult)) { @@ -246,7 +243,7 @@ private static boolean isFound(List selectErrors, SQLQueryError t public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { if (deleteResult.hasEmptyErrors()) { - if (selectResult.hasErrors()) { + if (!selectResult.hasEmptyErrors()) { return "SELECT has errors, but DELETE does not."; } if (!selectResult.hasSameAccessedRows(deleteResult)) { @@ -459,7 +456,7 @@ private List getErrors() throws SQLException { } resultSet.close(); } - + Collections.sort(queryErrors); return queryErrors; } From df600ff1251471783739fa4d4fd2fbea7ba66fef Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Wed, 13 Aug 2025 13:55:42 +0800 Subject: [PATCH 710/856] Serialize StateToReproduce directly --- src/sqlancer/Main.java | 20 ++++++++- src/sqlancer/MainOptions.java | 7 ++++ src/sqlancer/StateToReproduce.java | 67 ++++++++++++++++++++++++++++-- 3 files changed, 90 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index eb3ad7f0c..87023b0a2 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -5,6 +5,7 @@ import java.io.IOException; import java.io.Writer; import java.nio.file.Files; +import java.nio.file.Path; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.ArrayList; @@ -78,6 +79,7 @@ public static final class StateLogger { public FileWriter currentFileWriter; private FileWriter queryPlanFileWriter; private FileWriter reduceFileWriter; + private Path reproduceFilePath; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; @@ -127,7 +129,13 @@ public StateLogger(String databaseName, DatabaseProvider provider, Main reduceFileDir.mkdir(); } this.reduceFile = new File(reduceFileDir, databaseName + "-reduce.log"); - + } + if (options.serializeReproduceState()) { + File reproduceFileDir = new File(dir, "reproduce"); + if (!reproduceFileDir.exists()) { + reproduceFileDir.mkdir(); + } + reproduceFilePath = new File(reproduceFileDir, databaseName + ".ser").toPath(); } this.databaseProvider = provider; } @@ -341,6 +349,10 @@ private String removeNamesFromQueryPlans(String queryPlan) { result = result.replaceAll("i[0-9]+", "i0"); // Avoid duplicate indexes return result + "\n"; } + + public Path getReproduceFilePath() { + return reproduceFilePath; + } } public static class QueryManager { @@ -461,6 +473,9 @@ public void run() throws Exception { throw new AssertionError(e); } + if (options.serializeReproduceState() && reproducer != null) { + stateToRepro.serialize(logger.getReproduceFilePath()); + } if (options.reduceAST() && !options.useReducer()) { throw new AssertionError("To reduce AST, use-reducer option must be enabled first"); } @@ -675,6 +690,9 @@ private boolean run(MainOptions options, ExecutorService execService, executor.getStateToReproduce().exception = reduce.getMessage(); executor.getLogger().logFileWriter = null; executor.getLogger().logException(reduce, executor.getStateToReproduce()); + if (options.serializeReproduceState()) { + executor.getStateToReproduce().serialize(executor.getLogger().getReproduceFilePath()); + } return false; } finally { try { diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index a6a252fa3..a5142fcf0 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -123,6 +123,9 @@ public class MainOptions { @Parameter(names = "--database-prefix", description = "The prefix used for each database created") private String databasePrefix = "database"; // NOPMD + @Parameter(names = "--serialize-reproduce-state", description = "Serialize the state to reproduce") + private boolean serializeReproduceState = false; // NOPMD + @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") private boolean useReducer = false; // NOPMD @@ -304,6 +307,10 @@ public boolean performConnectionTest() { return useConnectionTest; } + public boolean serializeReproduceState() { + return serializeReproduceState; + } + public boolean useReducer() { return useReducer; } diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 33bfe3398..a8586745f 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -1,19 +1,26 @@ package sqlancer; import java.io.Closeable; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.List; import sqlancer.common.query.Query; -public class StateToReproduce { +public class StateToReproduce implements Serializable { + private static final long serialVersionUID = 1L; - private List> statements = new ArrayList<>(); + private transient List> statements = new ArrayList<>(); private final String databaseName; - private final DatabaseProvider databaseProvider; + private transient DatabaseProvider databaseProvider; public String databaseVersion; @@ -131,6 +138,60 @@ public OracleRunReproductionState createLocalState() { return new OracleRunReproductionState(); } + public void serialize(Path path) { + try (ObjectOutputStream oos = new ObjectOutputStream(Files.newOutputStream(path))) { + oos.writeObject(this); + } catch (IOException e) { + throw new AssertionError(e); + } + } + + public static StateToReproduce deserialize(Path path) { + try (ObjectInputStream ois = new ObjectInputStream(Files.newInputStream(path))) { + return (StateToReproduce) ois.readObject(); + } catch (IOException | ClassNotFoundException e) { + throw new AssertionError(e); + } + } + + private void writeObject(ObjectOutputStream out) throws IOException { + out.defaultWriteObject(); + + List statementStrings = new ArrayList<>(); + for (Query q : this.statements) { + statementStrings.add(q.getLogString()); + } + out.writeObject(statementStrings); + out.writeObject(this.databaseProvider != null ? this.databaseProvider.getDBMSName() : null); + } + + @SuppressWarnings("unchecked") + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { + in.defaultReadObject(); + this.statements = new ArrayList<>(); + List statementStrings = (List) in.readObject(); + String dbmsName = (String) in.readObject(); + + DatabaseProvider provider = null; + if (dbmsName != null) { + List> providers = Main.getDBMSProviders(); + for (DatabaseProvider p : providers) { + if (p.getDBMSName().equals(dbmsName)) { + provider = p; + break; + } + } + } + + if (provider == null || statementStrings == null) { + throw new AssertionError("Database provider or statement is null"); + } + this.databaseProvider = provider; + for (String s : statementStrings) { + this.statements.add(provider.getLoggableFactory().getQueryForStateToReproduce(s)); + } + } + public void setStatements(List> statements) { this.statements = statements; } From cfc7877f2d4a292284206b35f5fd1ad748fabe66 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Fri, 15 Aug 2025 14:31:53 +0800 Subject: [PATCH 711/856] Serialize full List> in StateToReproduce --- pom.xml | 1 + src/sqlancer/StateToReproduce.java | 17 +---------------- src/sqlancer/common/log/Loggable.java | 4 +++- src/sqlancer/common/query/ExpectedErrors.java | 4 +++- src/sqlancer/common/query/SQLQueryAdapter.java | 4 +++- 5 files changed, 11 insertions(+), 19 deletions(-) diff --git a/pom.xml b/pom.xml index 734b44f5d..3292576a8 100644 --- a/pom.xml +++ b/pom.xml @@ -114,6 +114,7 @@ 11 ${project.basedir}/.settings/org.eclipse.jdt.core.prefs + -Xlint:-serial eclipse true diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index a8586745f..0e5bd1816 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -16,7 +16,7 @@ public class StateToReproduce implements Serializable { private static final long serialVersionUID = 1L; - private transient List> statements = new ArrayList<>(); + private List> statements = new ArrayList<>(); private final String databaseName; @@ -157,19 +157,11 @@ public static StateToReproduce deserialize(Path path) { private void writeObject(ObjectOutputStream out) throws IOException { out.defaultWriteObject(); - List statementStrings = new ArrayList<>(); - for (Query q : this.statements) { - statementStrings.add(q.getLogString()); - } - out.writeObject(statementStrings); out.writeObject(this.databaseProvider != null ? this.databaseProvider.getDBMSName() : null); } - @SuppressWarnings("unchecked") private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException { in.defaultReadObject(); - this.statements = new ArrayList<>(); - List statementStrings = (List) in.readObject(); String dbmsName = (String) in.readObject(); DatabaseProvider provider = null; @@ -182,14 +174,7 @@ private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundE } } } - - if (provider == null || statementStrings == null) { - throw new AssertionError("Database provider or statement is null"); - } this.databaseProvider = provider; - for (String s : statementStrings) { - this.statements.add(provider.getLoggableFactory().getQueryForStateToReproduce(s)); - } } public void setStatements(List> statements) { diff --git a/src/sqlancer/common/log/Loggable.java b/src/sqlancer/common/log/Loggable.java index 0d5fef218..7796009ee 100644 --- a/src/sqlancer/common/log/Loggable.java +++ b/src/sqlancer/common/log/Loggable.java @@ -1,5 +1,7 @@ package sqlancer.common.log; -public interface Loggable { +import java.io.Serializable; + +public interface Loggable extends Serializable { String getLogString(); } diff --git a/src/sqlancer/common/query/ExpectedErrors.java b/src/sqlancer/common/query/ExpectedErrors.java index 38d5c9757..eb281efbe 100644 --- a/src/sqlancer/common/query/ExpectedErrors.java +++ b/src/sqlancer/common/query/ExpectedErrors.java @@ -1,5 +1,6 @@ package sqlancer.common.query; +import java.io.Serializable; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -11,7 +12,8 @@ * result in an error "UNIQUE constraint violated" when it attempts to insert a duplicate value in a column declared as * UNIQUE. */ -public class ExpectedErrors { +public class ExpectedErrors implements Serializable { + private static final long serialVersionUID = 1L; private final Set errors; private final Set regexes; diff --git a/src/sqlancer/common/query/SQLQueryAdapter.java b/src/sqlancer/common/query/SQLQueryAdapter.java index 3784ed635..22cb704f8 100644 --- a/src/sqlancer/common/query/SQLQueryAdapter.java +++ b/src/sqlancer/common/query/SQLQueryAdapter.java @@ -1,5 +1,6 @@ package sqlancer.common.query; +import java.io.Serializable; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; @@ -9,7 +10,8 @@ import sqlancer.Main; import sqlancer.SQLConnection; -public class SQLQueryAdapter extends Query { +public class SQLQueryAdapter extends Query implements Serializable { + private static final long serialVersionUID = 1L; private final String query; private final ExpectedErrors expectedErrors; From 30d879acb9f1919ac8f8cbd622bda31e3edfb826 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Sun, 17 Aug 2025 16:32:52 +0800 Subject: [PATCH 712/856] Add serialVersionUID --- pom.xml | 1 - src/sqlancer/cnosdb/query/CnosDBOtherQuery.java | 2 ++ src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java | 1 + src/sqlancer/cnosdb/query/CnosDBSelectQuery.java | 1 + src/sqlancer/common/log/LoggedString.java | 1 + src/sqlancer/common/query/Query.java | 1 + src/sqlancer/common/query/SQLQueryResultCheckAdapter.java | 1 + src/sqlancer/postgres/gen/PostgresDiscardGenerator.java | 1 + src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java | 1 + test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java | 1 + 10 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3292576a8..734b44f5d 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,6 @@ 11 ${project.basedir}/.settings/org.eclipse.jdt.core.prefs - -Xlint:-serial eclipse true diff --git a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java index bd48c7b7e..f0a37056c 100644 --- a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java +++ b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java @@ -6,6 +6,8 @@ import sqlancer.common.query.ExpectedErrors; public class CnosDBOtherQuery extends CnosDBQueryAdapter { + private static final long serialVersionUID = 1L; + public CnosDBOtherQuery(String query, ExpectedErrors errors) { super(query, errors); } diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java index 2b2dda5e6..115f96ffc 100644 --- a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java +++ b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java @@ -5,6 +5,7 @@ import sqlancer.common.query.Query; public abstract class CnosDBQueryAdapter extends Query { + private static final long serialVersionUID = 1L; String query; ExpectedErrors errors; diff --git a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java index ca7f61c3a..1c9228182 100644 --- a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java +++ b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java @@ -7,6 +7,7 @@ import sqlancer.common.query.SQLancerResultSet; public class CnosDBSelectQuery extends CnosDBQueryAdapter { + private static final long serialVersionUID = 1L; CnosDBResultSet resultSet; public CnosDBSelectQuery(String query, ExpectedErrors errors) { diff --git a/src/sqlancer/common/log/LoggedString.java b/src/sqlancer/common/log/LoggedString.java index 4f449034f..696203a78 100644 --- a/src/sqlancer/common/log/LoggedString.java +++ b/src/sqlancer/common/log/LoggedString.java @@ -1,6 +1,7 @@ package sqlancer.common.log; public class LoggedString implements Loggable { + private static final long serialVersionUID = 1L; private final String loggedString; diff --git a/src/sqlancer/common/query/Query.java b/src/sqlancer/common/query/Query.java index 44efc0084..ca90619c7 100644 --- a/src/sqlancer/common/query/Query.java +++ b/src/sqlancer/common/query/Query.java @@ -5,6 +5,7 @@ import sqlancer.common.log.Loggable; public abstract class Query implements Loggable { + private static final long serialVersionUID = 1L; /** * Gets the query string, which is guaranteed to be terminated with a semicolon. diff --git a/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java index 8f6a01a4b..1eaae2424 100644 --- a/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java +++ b/src/sqlancer/common/query/SQLQueryResultCheckAdapter.java @@ -9,6 +9,7 @@ import sqlancer.SQLConnection; public class SQLQueryResultCheckAdapter extends SQLQueryAdapter { + private static final long serialVersionUID = 1L; private final Consumer rsChecker; diff --git a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java index c977fa272..0c02efcd0 100644 --- a/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDiscardGenerator.java @@ -25,6 +25,7 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { } sb.append(what); return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { + private static final long serialVersionUID = 1L; @Override public boolean couldAffectSchema() { diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java index 40ccdfa0d..156fb97da 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDiscardGenerator.java @@ -25,6 +25,7 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { } sb.append(what); return new SQLQueryAdapter(sb.toString(), ExpectedErrors.from("cannot run inside a transaction block")) { + private static final long serialVersionUID = 1L; @Override public boolean couldAffectSchema() { diff --git a/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java index 37389109f..9767723e0 100644 --- a/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java +++ b/test/sqlancer/reducer/VirtualDB/VirtualDBQuery.java @@ -7,6 +7,7 @@ import java.sql.SQLException; public class VirtualDBQuery extends SQLQueryAdapter { + private static final long serialVersionUID = 1L; public VirtualDBQuery(String query) { // Since the base class must check the format From a28ede83778842b4468a814ebcf3a9670263597f Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Wed, 20 Aug 2025 13:43:07 +0800 Subject: [PATCH 713/856] Add serialization tests --- src/sqlancer/StateToReproduce.java | 4 + test/sqlancer/TestStateToReproduce.java | 109 ++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 test/sqlancer/TestStateToReproduce.java diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index 0e5bd1816..c849b535d 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -47,6 +47,10 @@ public String getDatabaseVersion() { return databaseVersion; } + public DatabaseProvider getDatabaseProvider() { + return databaseProvider; + } + /** * Logs the statement string without executing the corresponding statement. * diff --git a/test/sqlancer/TestStateToReproduce.java b/test/sqlancer/TestStateToReproduce.java new file mode 100644 index 000000000..6fcdc22f1 --- /dev/null +++ b/test/sqlancer/TestStateToReproduce.java @@ -0,0 +1,109 @@ +package sqlancer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.Query; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.sqlite3.SQLite3Provider; + +public class TestStateToReproduce { + + @TempDir + Path tempDir; + + @Test + public void testBasicFields() throws IOException { + SQLite3Provider provider = new SQLite3Provider(); + StateToReproduce state = new StateToReproduce("test_db", provider); + state.databaseVersion = "3.36.0"; + state.seedValue = 12345L; + state.exception = "Test exception message"; + + Path file = tempDir.resolve("test_basic.ser"); + state.serialize(file); + StateToReproduce result = StateToReproduce.deserialize(file); + + assertEquals(state.getDatabaseName(), result.getDatabaseName()); + assertEquals(state.getDatabaseVersion(), result.getDatabaseVersion()); + assertEquals(state.getSeedValue(), result.getSeedValue()); + assertEquals(state.getException(), result.getException()); + } + + @Test + public void testStatements() throws IOException { + SQLite3Provider provider = new SQLite3Provider(); + StateToReproduce state = new StateToReproduce("test_statements", provider); + List> statements = new ArrayList<>(); + + ExpectedErrors errors1 = new ExpectedErrors(); + errors1.add("syntax error"); + errors1.add("table already exists"); + statements.add(new SQLQueryAdapter("CREATE TABLE test (id INTEGER);", errors1)); + + ExpectedErrors errors2 = new ExpectedErrors(); + errors2.add("constraint failed"); + statements.add(new SQLQueryAdapter("INSERT INTO test VALUES (1);", errors2)); + + statements.add(new SQLQueryAdapter("SELECT * FROM test;", new ExpectedErrors())); + state.setStatements(statements); + + Path file = tempDir.resolve("test_statements.ser"); + state.serialize(file); + StateToReproduce result = StateToReproduce.deserialize(file); + + List> resultStatements = result.getStatements(); + assertEquals(3, resultStatements.size()); + + Query q1 = resultStatements.get(0); + Query q2 = resultStatements.get(1); + Query q3 = resultStatements.get(2); + + assertEquals("CREATE TABLE test (id INTEGER);", q1.getLogString()); + assertEquals("INSERT INTO test VALUES (1);", q2.getLogString()); + assertEquals("SELECT * FROM test;", q3.getLogString()); + + ExpectedErrors e1 = q1.getExpectedErrors(); + ExpectedErrors e2 = q2.getExpectedErrors(); + ExpectedErrors e3 = q3.getExpectedErrors(); + + assertTrue(e1.errorIsExpected("syntax error")); + assertTrue(e1.errorIsExpected("table already exists")); + assertFalse(e1.errorIsExpected("constraint failed")); + + assertFalse(e2.errorIsExpected("syntax error")); + assertTrue(e2.errorIsExpected("constraint failed")); + + assertFalse(e3.errorIsExpected("syntax error")); + assertFalse(e3.errorIsExpected("constraint failed")); + } + + @Test + public void testDatabaseProvider() throws IOException { + SQLite3Provider provider = new SQLite3Provider(); + StateToReproduce state = new StateToReproduce("test_provider", provider); + state.logStatement("CREATE TABLE test (id INTEGER);"); + + Path file = tempDir.resolve("test_provider.ser"); + state.serialize(file); + StateToReproduce result = StateToReproduce.deserialize(file); + + // Verify databaseProvider is correctly deserialized + assertEquals("sqlite3", result.getDatabaseProvider().getDBMSName()); + + // Verify databaseProvider functionality by testing logStatement + result.logStatement("INSERT INTO test VALUES (1);"); + assertEquals(2, result.getStatements().size()); + assertEquals("INSERT INTO test VALUES (1);", result.getStatements().get(1).getLogString()); + } +} \ No newline at end of file From 150d2d53ca27074eea3de7c3ae312d83248056ed Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Sun, 24 Aug 2025 22:39:52 +0800 Subject: [PATCH 714/856] Fix StateToReproduce serialization --- src/sqlancer/Main.java | 1 + src/sqlancer/StateToReproduce.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 87023b0a2..1f2642f95 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -691,6 +691,7 @@ private boolean run(MainOptions options, ExecutorService execService, executor.getLogger().logFileWriter = null; executor.getLogger().logException(reduce, executor.getStateToReproduce()); if (options.serializeReproduceState()) { + executor.getStateToReproduce().logStatement(reduce.getMessage()); // add the error statement executor.getStateToReproduce().serialize(executor.getLogger().getReproduceFilePath()); } return false; diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index c849b535d..e44d0ccf6 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -28,7 +28,7 @@ public class StateToReproduce implements Serializable { String exception; - public OracleRunReproductionState localState; + public transient OracleRunReproductionState localState; public StateToReproduce(String databaseName, DatabaseProvider databaseProvider) { this.databaseName = databaseName; From f7310f96163a05730db51e9009ac174cfc4689e3 Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Sun, 24 Aug 2025 22:49:21 +0800 Subject: [PATCH 715/856] Add a standalone reducer --- src/sqlancer/StandaloneReducer.java | 130 ++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 src/sqlancer/StandaloneReducer.java diff --git a/src/sqlancer/StandaloneReducer.java b/src/sqlancer/StandaloneReducer.java new file mode 100644 index 000000000..dfc0cc348 --- /dev/null +++ b/src/sqlancer/StandaloneReducer.java @@ -0,0 +1,130 @@ +package sqlancer; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.Query; + +/** + * A standalone tool to reduce bug-triggering SQL statements using the delta debugging algorithm. + */ +public class StandaloneReducer { + private int partitionNum = 2; + private final StateToReproduce originalState; + private final DatabaseProvider databaseProvider; + + public StandaloneReducer(Path serFilePath) throws Exception { + this.originalState = StateToReproduce.deserialize(serFilePath); + this.databaseProvider = originalState.getDatabaseProvider(); + if (this.databaseProvider == null) { + throw new IllegalStateException("Failed to get database provider from .ser file"); + } + } + + /** + * Performs the main reduction algorithm using partition-based delta debugging. + * + * @return List of reduced SQL statements that still trigger bugs. + */ + public List> reduce() throws Exception { + List> queries = new ArrayList<>(originalState.getStatements()); + if (queries.size() <= 1) { + return queries; + } + + partitionNum = 2; + while (queries.size() >= 2) { + boolean changedInThisPass = false; + List> result = tryReduction(queries); + + if (result.size() < queries.size()) { + queries = result; + changedInThisPass = true; + } + + if (changedInThisPass) { + partitionNum = 2; + } else { + if (partitionNum >= queries.size()) { + break; + } + partitionNum = Math.min(partitionNum * 2, queries.size()); + } + } + + System.out.println("Reduction completed successfully!"); + System.out.println("Final size: " + queries.size() + " statements (" + + String.format("%.1f", (1.0 - (double) queries.size() / originalState.getStatements().size()) * 100) + + "% reduction)"); + System.out.println("Final queries:"); + for (Query query : queries) { + System.out.println(query.getQueryString()); + } + + return queries; + } + + private List> tryReduction(List> queries) throws Exception { + int start = 0; + int subLength = queries.size() / partitionNum; + + while (start < queries.size()) { + List> candidateQueries = new ArrayList<>(queries); + int endPoint = Math.min(start + subLength, candidateQueries.size()); + candidateQueries.subList(start, endPoint).clear(); + + if (testExceptionStillExists(candidateQueries)) { + return candidateQueries; + } + + start += subLength; + } + + return queries; + } + + // Test if bug still exists with reduced query set + @SuppressWarnings("unchecked") + private , O extends DBMSSpecificOptions, C extends SQLancerDBConnection> boolean testExceptionStillExists( + List> queries) { + try { + DatabaseProvider typedProvider = (DatabaseProvider) databaseProvider; + G globalState = typedProvider.getGlobalStateClass().getDeclaredConstructor().newInstance(); + + try (C connection = typedProvider.createDatabase(globalState)) { + globalState.setConnection(connection); + for (Query query : queries) { + try { + Query typedQuery = (Query) query; + typedQuery.execute(globalState); + } catch (Throwable e) { + // Any exception is considered a success + return true; + } + } + // No exception occurred + return false; + } + } catch (Throwable e) { + return true; + } + } + + public static void main(String[] args) { + try { + if (args.length == 0) { + System.err.println( + "Usage: java -cp target/sqlancer-2.0.0.jar sqlancer.StandaloneReducer "); + System.exit(1); + } + StandaloneReducer reducer = new StandaloneReducer(Paths.get(args[0])); + reducer.reduce(); + } catch (Throwable e) { + System.err.println("ERROR: " + e.getMessage()); + e.printStackTrace(); + System.exit(1); + } + } +} From 513d0d8a15fb356c85198cdce2370862fc3935af Mon Sep 17 00:00:00 2001 From: Imcaicai <2939361916@qq.com> Date: Mon, 25 Aug 2025 13:06:51 +0800 Subject: [PATCH 716/856] Write final queries to a file --- src/sqlancer/StandaloneReducer.java | 30 +++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/sqlancer/StandaloneReducer.java b/src/sqlancer/StandaloneReducer.java index dfc0cc348..813160060 100644 --- a/src/sqlancer/StandaloneReducer.java +++ b/src/sqlancer/StandaloneReducer.java @@ -1,5 +1,7 @@ package sqlancer; +import java.io.FileWriter; +import java.io.PrintWriter; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; @@ -14,13 +16,16 @@ public class StandaloneReducer { private int partitionNum = 2; private final StateToReproduce originalState; private final DatabaseProvider databaseProvider; + private final Path outputPath; - public StandaloneReducer(Path serFilePath) throws Exception { - this.originalState = StateToReproduce.deserialize(serFilePath); + public StandaloneReducer(Path inputPath, Path outputPath) throws Exception { + this.originalState = StateToReproduce.deserialize(inputPath); this.databaseProvider = originalState.getDatabaseProvider(); if (this.databaseProvider == null) { throw new IllegalStateException("Failed to get database provider from .ser file"); } + this.outputPath = outputPath != null ? outputPath + : Paths.get(inputPath.toString().replaceAll("\\.ser$", ".sql")); } /** @@ -54,14 +59,15 @@ public List> reduce() throws Exception { } } - System.out.println("Reduction completed successfully!"); + try (PrintWriter writer = new PrintWriter(new FileWriter(outputPath.toFile()))) { + for (Query query : queries) { + writer.println(query.getQueryString()); + } + } + System.out.println("Reduction completed successfully! SQL statements written to: " + outputPath.toString()); System.out.println("Final size: " + queries.size() + " statements (" + String.format("%.1f", (1.0 - (double) queries.size() / originalState.getStatements().size()) * 100) + "% reduction)"); - System.out.println("Final queries:"); - for (Query query : queries) { - System.out.println(query.getQueryString()); - } return queries; } @@ -100,7 +106,8 @@ private List> tryReduction(List> queries) throws Exception { Query typedQuery = (Query) query; typedQuery.execute(globalState); } catch (Throwable e) { - // Any exception is considered a success + // Any exception not declared as an expected error by the query indicates that an (unexpected) + // exception still exists return true; } } @@ -116,10 +123,13 @@ public static void main(String[] args) { try { if (args.length == 0) { System.err.println( - "Usage: java -cp target/sqlancer-2.0.0.jar sqlancer.StandaloneReducer "); + "Usage: java -cp target/sqlancer-2.0.0.jar sqlancer.StandaloneReducer [output-file]"); System.exit(1); } - StandaloneReducer reducer = new StandaloneReducer(Paths.get(args[0])); + Path inputPath = Paths.get(args[0]); + Path outputPath = args.length > 1 ? Paths.get(args[1]) : null; + + StandaloneReducer reducer = new StandaloneReducer(inputPath, outputPath); reducer.reduce(); } catch (Throwable e) { System.err.println("ERROR: " + e.getMessage()); From 6a43fe08183ef9602f1c886733d5f1e24fb3d501 Mon Sep 17 00:00:00 2001 From: luliqwerty <3535768174@qq.com> Date: Wed, 27 Aug 2025 13:06:29 +0800 Subject: [PATCH 717/856] doc(README): Add DQE Oracle Testing Method in README --- README.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index f88b77faa..134f47666 100644 --- a/README.md +++ b/README.md @@ -75,15 +75,16 @@ Usage: SQLancer [options] [command] [command options] SQLancer has pioneered and includes multiple approaches for DBMS testing, as outlined below in chronological order. -| Technique | Venue | Links | Description | -|-----------------------------------------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Pivoted Query Synthesis (PQS) | OSDI 2020 | [Paper](https://www.usenix.org/system/files/osdi20-rigger.pdf) [Video](https://www.youtube.com/watch?v=0aeDyXgzo04 ) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described here. PQS effectively detects bugs, but requires more implementation effort than other testing approaches that follow a metamorphic testing or differential testing methodology. Thus, it is currently unmaintained. | -| Non-optimizing Reference Engine Construction (NoREC) | ESEC/FSE 2020 | [Paper](https://arxiv.org/abs/2007.08292) [Video](https://www.youtube.com/watch?v=4mbzytrWJhQ) | NoREC aims to find optimization bugs. It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. The approach applies primarily to simple queries with a filter predicate. | -| Ternary Logic Partitioning (TLP) | OOPSLA 2020 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3428279) [Video](https://www.youtube.com/watch?v=FN9OLbGh0VI) | TLP partitions a query into three partitioning queries, whose results are composed and compared to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. It is among the most widely adopted testing techniques. | -| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | QPG is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | -| Cardinality Estimation Restriction Testing (CERT) | ICSE 2024 | [Paper](https://arxiv.org/pdf/2306.00355) [Code](https://github.com/sqlancer/sqlancer/issues/822) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. From a given input query, it derives a more restrictive query, whose estimated cardinality should be no more than that of the original query. A violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. CERT is the only test oracle that is part of SQLancer that was designed to find performance issues. | -| Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | -| Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | +| Technique | Venue | Links | Description | +|-----------------------------------------------------------------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Pivoted Query Synthesis (PQS) | OSDI 2020 | [Paper](https://www.usenix.org/system/files/osdi20-rigger.pdf) [Video](https://www.youtube.com/watch?v=0aeDyXgzo04 ) | PQS is the first technique that we designed and implemented. It randomly selects a row, called a pivot row, for which a query is generated that is guaranteed to fetch the row. If the row is not contained in the result set, a bug has been detected. It is fully described here. PQS effectively detects bugs, but requires more implementation effort than other testing approaches that follow a metamorphic testing or differential testing methodology. Thus, it is currently unmaintained. | +| Non-optimizing Reference Engine Construction (NoREC) | ESEC/FSE 2020 | [Paper](https://arxiv.org/abs/2007.08292) [Video](https://www.youtube.com/watch?v=4mbzytrWJhQ) | NoREC aims to find optimization bugs. It translates a query that is potentially optimized by the DBMS to one for which hardly any optimizations are applicable, and compares the two result sets. A mismatch between the result sets indicates a bug in the DBMS. The approach applies primarily to simple queries with a filter predicate. | +| Ternary Logic Partitioning (TLP) | OOPSLA 2020 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3428279) [Video](https://www.youtube.com/watch?v=FN9OLbGh0VI) | TLP partitions a query into three partitioning queries, whose results are composed and compared to the original query's result set. A mismatch in the result sets indicates a bug in the DBMS. In contrast to NoREC and PQS, it can detect bugs in advanced features such as aggregate functions. It is among the most widely adopted testing techniques. | +| Differential Query Execution (DQE) | ICSE 2023 | [Paper](https://ieeexplore.ieee.org/document/10172736) [Code](https://github.com/sqlancer/sqlancer/pull/1251) | Differential Query Execution (DQE) is a novel and general approach to detect logic bugs in SELECT, UPDATE and DELETE queries. DQE solves the test oracle problem by executing SELECT, UPDATE and DELETE queries with the same predicate φ, and observing inconsistencies among their execution results. For example, if a row that is updated by an UPDATE query with a predicate φ does not appear in the query result of a SELECT query with the same predicate φ, a logic bug is detected in the target DBMS. We append two extra columns to each table in a database to uniquely identify each row and track whether a row has been modified. We further rewrite SELECT and UPDATE queries to identify their accessed rows. DQE supports MySQL. | +| Query Plan Guidance (QPG) | ICSE 2023 | [Paper](https://arxiv.org/pdf/2312.17510) [Video](https://youtu.be/6EjQ1cKiZJU?si=gh7uoykRqNjl3GXR&t=1820) [Code](https://github.com/sqlancer/sqlancer/issues/641) | QPG is a feedback-guided test case generation approach. It is based on the insights that query plans capture whether interesting behavior is exercised within the DBMS. It works by mutating the database state when no new query plans have been observed after executing a number of queries, expecting that the new state enables new query plans to be triggered. This approach is enabled by option `--qpg-enable` and supports TLP and NoREC oracles for SQLite, CockroachDB, TiDB, and Materialize. It is the only approach that specifically tackles the test input generation problem. | +| Cardinality Estimation Restriction Testing (CERT) | ICSE 2024 | [Paper](https://arxiv.org/pdf/2306.00355) [Code](https://github.com/sqlancer/sqlancer/issues/822) | CERT aims to find performance issues through unexpected estimated cardinalities, which represent the estimated number of returned rows. From a given input query, it derives a more restrictive query, whose estimated cardinality should be no more than that of the original query. A violation indicates a potential performance issue. CERT supports TiDB, CockroachDB, and MySQL. CERT is the only test oracle that is part of SQLancer that was designed to find performance issues. | +| Differential Query Plans (DQP) | SIGMOD 2024 | [Paper](https://dl.acm.org/doi/pdf/10.1145/3654991) [Video](https://www.youtube.com/watch?v=9Qp7quJfGEk) [Code](https://github.com/sqlancer/sqlancer/issues/918) | DQP aims to find logic bugs by controlling the execution of different query plans for a given query and validating that they produce a consistent result. DQP supports MySQL, MariaDB, and TiDB. | +| Constant Optimization Driven Database System Testing (CODDTest) | SIGMOD 2025 | [Code](https://github.com/sqlancer/sqlancer/pull/1054) | CODDTest finds logic bugs in DBMSs, including in advanced features such as subqueries. It is based on the insight that we can assume the database state to be constant for a database session, which then enables us to substitute parts of a query with their results, essentially corresponding to constant folding and constant propagation, which are two traditional compiler optimizations. | Please find the `.bib` entries [here](docs/PAPERS.md). | From 8749a6ecb05f049deeacb8593de3fcd44bee92d0 Mon Sep 17 00:00:00 2001 From: luliqwerty <3535768174@qq.com> Date: Tue, 2 Sep 2025 08:29:04 +0800 Subject: [PATCH 718/856] feat(DQE): Add Extensibility to DQE --- src/sqlancer/common/oracle/DQEBase.java | 18 ++++++------- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 25 +++++++++++-------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/src/sqlancer/common/oracle/DQEBase.java b/src/sqlancer/common/oracle/DQEBase.java index ba9af3bd1..0d954b115 100644 --- a/src/sqlancer/common/oracle/DQEBase.java +++ b/src/sqlancer/common/oracle/DQEBase.java @@ -13,7 +13,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLQueryError; import sqlancer.common.schema.AbstractRelationalTable; -import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.common.schema.AbstractTables; /* * In DBMSs, SELECT, UPDATE and DELETE queries utilize predicates (i.e., WHERE clauses) to specify which rows to retrieve, update or delete, respectively. @@ -54,19 +54,15 @@ public DQEBase(S state) { this.options = state.getOptions(); } - public abstract String generateSelectStatement(MySQLTables tables, String tableName, String whereClauseStr); + public abstract String generateSelectStatement(AbstractTables tables, String tableName, + String whereClauseStr); - public abstract String generateUpdateStatement(MySQLTables tables, String tableName, String whereClauseStr); + public abstract String generateUpdateStatement(AbstractTables tables, String tableName, + String whereClauseStr); - public abstract String generateDeleteStatement(MySQLTables tables, String tableName, String whereClauseStr); + public abstract String generateDeleteStatement(String tableName, String whereClauseStr); - /** - * Add auxiliary columns to the database A abstract method, subclasses need to implement it. - * - * @param table - * - * @throws SQLException - */ + // Add auxiliary columns to the database A abstract method, subclasses need to implement it. public abstract void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException; public void dropAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index 4585fc830..de64b0415 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -22,6 +22,7 @@ import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractRelationalTable; import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTables; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema; @@ -57,16 +58,17 @@ public MySQLDQEOracle(MySQLGlobalState state) { } @Override - public String generateSelectStatement(MySQLTables tables, String tableName, String whereClauseStr) { + public String generateSelectStatement(AbstractTables tables, String tableName, String whereClauseStr) { operateOnSingleTable = tables.getTables().size() == 1; List selectColumns = new ArrayList<>(); - for (MySQLTable table : tables.getTables()) { + MySQLTables mySQLTables = (MySQLTables) tables; + for (MySQLTable table : mySQLTables.getTables()) { selectColumns.add(table.getName() + "." + COLUMN_ROWID); } if (operateOnSingleTable && Randomly.getBooleanWithSmallProbability()) { generateOrderBy = true; // generate order by columns - for (MySQLColumn column : Randomly.nonEmptySubset(tables.getColumns())) { + for (MySQLColumn column : Randomly.nonEmptySubset(mySQLTables.getColumns())) { orderColumns.add(column.getFullQualifiedName()); } @@ -88,9 +90,10 @@ public String generateSelectStatement(MySQLTables tables, String tableName, Stri } @Override - public String generateUpdateStatement(MySQLTables tables, String tableName, String whereClauseStr) { + public String generateUpdateStatement(AbstractTables tables, String tableName, String whereClauseStr) { List updateColumns = new ArrayList<>(); - for (MySQLTable table : tables.getTables()) { + MySQLTables mySQLTables = (MySQLTables) tables; + for (MySQLTable table : mySQLTables.getTables()) { updateColumns.add(String.format("%s = 1", table.getName() + "." + COLUMN_UPDATED)); } String updateStmt = String.format("UPDATE %s SET %s WHERE %s", tableName, Strings.join(",", updateColumns), @@ -105,7 +108,7 @@ public String generateUpdateStatement(MySQLTables tables, String tableName, Stri } @Override - public String generateDeleteStatement(MySQLTables tables, String tableName, String whereClauseStr) { + public String generateDeleteStatement(String tableName, String whereClauseStr) { String deleteStmt; if (operateOnSingleTable) { deleteStmt = String.format("DELETE FROM %s WHERE %s", tableName, whereClauseStr); @@ -141,7 +144,7 @@ public void check() throws SQLException { String updateStmt = generateUpdateStatement(tables, tableName, whereClauseStr); - String deleteStmt = generateDeleteStatement(tables, tableName, whereClauseStr); + String deleteStmt = generateDeleteStatement(tableName, whereClauseStr); for (MySQLTable table : tables.getTables()) { addAuxiliaryColumns(table); @@ -201,7 +204,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); - if (!isFound(selectErrors, updateError)) { + if (notFound(selectErrors, updateError)) { return "SELECT has different errors from UPDATE."; } } @@ -228,7 +231,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult * * @return is targetError found in selectQueryErrors */ - private static boolean isFound(List selectErrors, SQLQueryError targetError) { + private static boolean notFound(List selectErrors, SQLQueryError targetError) { boolean found = false; for (int i = 0; i < selectErrors.size(); i++) { SQLQueryError selectError = selectErrors.get(i); @@ -238,7 +241,7 @@ private static boolean isFound(List selectErrors, SQLQueryError t break; } } - return found; + return !found; } public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { @@ -263,7 +266,7 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); - if (!isFound(selectErrors, deleteError)) { + if (notFound(selectErrors, deleteError)) { return "SELECT has different errors from DELETE."; } } From d76843182e205d154afb04afbc355245d47c934c Mon Sep 17 00:00:00 2001 From: luliqwerty <3535768174@qq.com> Date: Tue, 2 Sep 2025 16:05:57 +0800 Subject: [PATCH 719/856] refactor: rename notFound method in MySQLDQEOracle.java --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index de64b0415..8ddb6f315 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -204,7 +204,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); - if (notFound(selectErrors, updateError)) { + if (!isFound(selectErrors, updateError)) { return "SELECT has different errors from UPDATE."; } } @@ -231,7 +231,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult * * @return is targetError found in selectQueryErrors */ - private static boolean notFound(List selectErrors, SQLQueryError targetError) { + private static boolean isFound(List selectErrors, SQLQueryError targetError) { boolean found = false; for (int i = 0; i < selectErrors.size(); i++) { SQLQueryError selectError = selectErrors.get(i); @@ -241,7 +241,7 @@ private static boolean notFound(List selectErrors, SQLQueryError break; } } - return !found; + return found; } public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { @@ -266,7 +266,7 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); - if (notFound(selectErrors, deleteError)) { + if (!isFound(selectErrors, deleteError)) { return "SELECT has different errors from DELETE."; } } From d04835346e1d02b3a7fdb513dc5f62554cfe1b37 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Mon, 15 Sep 2025 14:55:18 +0800 Subject: [PATCH 720/856] tidb: fix wrong type judgement and analyze index Signed-off-by: Weizhen Wang --- .github/workflows/main.yml | 8 ++++---- src/sqlancer/tidb/TiDBSchema.java | 3 +++ src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java | 11 +++++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7b0e4e410..5c53192aa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -609,8 +609,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:v8.5.2 - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 + docker pull hawkingrei/tidb-playground:nightly-2025-09-16 + docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" @@ -636,8 +636,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Set up TiDB run: | - docker pull pingcap/tidb:v8.5.2 - docker run --name tidb-server -d -p 4000:4000 pingcap/tidb:v8.5.2 + docker pull hawkingrei/tidb-playground:nightly-2025-09-16 + docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 sleep 10 - name: Create SQLancer user run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 7c808beee..32e00504b 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -251,17 +251,20 @@ private static TiDBCompositeDataType getColumnType(String typeString) { primitiveType = TiDBDataType.INT; size = 1; break; + case "tinyint": case "tinyint(2)": case "tinyint(3)": case "tinyint(4)": primitiveType = TiDBDataType.INT; size = 1; break; + case "smallint": case "smallint(5)": case "smallint(6)": primitiveType = TiDBDataType.INT; size = 2; break; + case "int": case "int(10)": case "int(11)": primitiveType = TiDBDataType.INT; diff --git a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java index 8b43c635a..ede857dc5 100644 --- a/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBAnalyzeTableGenerator.java @@ -1,10 +1,12 @@ package sqlancer.tidb.gen; import java.sql.SQLException; +import java.util.List; import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.TableIndex; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBTable; @@ -17,12 +19,17 @@ private TiDBAnalyzeTableGenerator() { public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - boolean analyzeIndex = !table.getIndexes().isEmpty() && Randomly.getBoolean(); + List indexes = table.getIndexes(); + indexes.removeIf(index -> index.getIndexName().contains("PRIMARY")); + boolean analyzeIndex = !indexes.isEmpty() && Randomly.getBoolean(); StringBuilder sb = new StringBuilder("ANALYZE TABLE "); sb.append(table.getName()); if (analyzeIndex) { sb.append(" INDEX "); - sb.append(table.getRandomIndex().getIndexName()); + sb.append(Randomly.fromList(indexes).getIndexName()); + } + if (!analyzeIndex && Randomly.getBoolean()) { + sb.append(" ALL COLUMNS"); } if (Randomly.getBoolean()) { sb.append(" WITH "); From f60bfedded7ccd53c8e0b5679dfc6680d0c0ec56 Mon Sep 17 00:00:00 2001 From: Weizhen Wang Date: Thu, 18 Sep 2025 10:51:56 +0800 Subject: [PATCH 721/856] tidb: skip Data too long errors Signed-off-by: Weizhen Wang --- src/sqlancer/tidb/TiDBErrors.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/tidb/TiDBErrors.java b/src/sqlancer/tidb/TiDBErrors.java index 4f45debce..528acd0bb 100644 --- a/src/sqlancer/tidb/TiDBErrors.java +++ b/src/sqlancer/tidb/TiDBErrors.java @@ -108,6 +108,9 @@ public static List getInsertErrors() { errors.add("is not valid for CHARACTER SET"); errors.add("for function inet_aton"); errors.add("'Empty pattern is invalid' from regexp"); + errors.add("Data too long for expression index"); + errors.add("Data too long for column"); + errors.add("Data Too Long"); return errors; } From c58cc468cca9a89de59add149aa7af845fbebfcc Mon Sep 17 00:00:00 2001 From: Mohab-Sobhy Date: Thu, 18 Sep 2025 23:25:09 +0300 Subject: [PATCH 722/856] Add DeleteGenerator for MariaDB --- src/sqlancer/mariadb/MariaDBProvider.java | 15 +-- .../mariadb/gen/MariaDBDeleteGenerator.java | 94 +++++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 48254b8ec..8f7eb78ad 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -19,13 +19,7 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; -import sqlancer.mariadb.gen.MariaDBIndexGenerator; -import sqlancer.mariadb.gen.MariaDBInsertGenerator; -import sqlancer.mariadb.gen.MariaDBSetGenerator; -import sqlancer.mariadb.gen.MariaDBTableAdminCommandGenerator; -import sqlancer.mariadb.gen.MariaDBTableGenerator; -import sqlancer.mariadb.gen.MariaDBTruncateGenerator; -import sqlancer.mariadb.gen.MariaDBUpdateGenerator; +import sqlancer.mariadb.gen.*; @AutoService(DatabaseProvider.class) public class MariaDBProvider extends SQLProviderAdapter { @@ -47,6 +41,7 @@ enum Action { SET, // TRUNCATE, // UPDATE, // + DELETE, } @Override @@ -77,6 +72,9 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { case CREATE_INDEX: nrPerformed = globalState.getRandomly().getInteger(0, 2); break; + case DELETE: + nrPerformed = 10; + break; case SET: nrPerformed = 20; break; @@ -140,6 +138,9 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { case SET: query = MariaDBSetGenerator.set(globalState.getRandomly(), options); break; + case DELETE: + query = MariaDBDeleteGenerator.delete(globalState.getSchema(), globalState.getRandomly()); + break; default: throw new AssertionError(nextAction); } diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java new file mode 100644 index 000000000..d1f3d2600 --- /dev/null +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -0,0 +1,94 @@ +package sqlancer.mariadb.gen; + +import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractTables; +import sqlancer.mariadb.MariaDBSchema; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; +import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.ast.MariaDBVisitor; + +import java.util.Collections; + +public final class MariaDBDeleteGenerator { + + private MariaDBDeleteGenerator() { + } + + public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { + MariaDBTable table = schema.getRandomTable(); + + MariaDBExpressionGenerator expressionGenerator = new MariaDBExpressionGenerator(r); + + AbstractTables tablesAndColumns = + new AbstractTables<>(Collections.singletonList(table)); + expressionGenerator.setTablesAndColumns(tablesAndColumns); + + ExpectedErrors errors = new ExpectedErrors(); + + errors.add("foreign key constraint fails"); + errors.add("cannot delete or update a parent row"); + errors.add("Data truncated"); + errors.add("Division by 0"); + errors.add("Incorrect value"); + + StringBuilder sb = new StringBuilder("DELETE"); + + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" LOW_PRIORITY"); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" QUICK"); + } + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" IGNORE"); + } + + sb.append(" FROM "); + sb.append(table.getName()); + + if (Randomly.getBoolean()) { + sb.append(" WHERE "); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(MariaDBVisitor.asString( + MariaDBExpressionGenerator.getRandomConstant(r) + )); + } else { + sb.append(MariaDBVisitor.asString( + expressionGenerator.getRandomExpression() + )); + } + } + + // ORDER BY + LIMIT + if (Randomly.getBooleanWithRatherLowProbability() && !table.getColumns().isEmpty()) { + sb.append(" ORDER BY "); + sb.append(Randomly.fromList(table.getColumns()).getName()); + if (Randomly.getBoolean()) { + sb.append(Randomly.getBoolean() ? " ASC" : " DESC"); + } + } + + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" LIMIT "); + sb.append(Randomly.getNotCachedInteger(1, 10)); + } + + // RETURNING clause (MariaDB >= 10.5) + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" RETURNING "); + if (Randomly.getBooleanWithRatherLowProbability()) { + sb.append(MariaDBVisitor.asString( + MariaDBExpressionGenerator.getRandomConstant(r) + )); + } else { + sb.append(MariaDBVisitor.asString( + expressionGenerator.getRandomExpression() + )); + } + } + + return new SQLQueryAdapter(sb.toString(), errors); + } +} From fde4da74ad9222ac537a6e9ad3f0bfd55daa95a8 Mon Sep 17 00:00:00 2001 From: Mohab-Sobhy Date: Wed, 8 Oct 2025 10:59:55 +0300 Subject: [PATCH 723/856] Format code to fix linter error --- .../mariadb/gen/MariaDBDeleteGenerator.java | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index d1f3d2600..28136eb08 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -21,8 +21,8 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { MariaDBExpressionGenerator expressionGenerator = new MariaDBExpressionGenerator(r); - AbstractTables tablesAndColumns = - new AbstractTables<>(Collections.singletonList(table)); + AbstractTables tablesAndColumns = new AbstractTables<>( + Collections.singletonList(table)); expressionGenerator.setTablesAndColumns(tablesAndColumns); ExpectedErrors errors = new ExpectedErrors(); @@ -51,13 +51,9 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { if (Randomly.getBoolean()) { sb.append(" WHERE "); if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString( - MariaDBExpressionGenerator.getRandomConstant(r) - )); + sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); } else { - sb.append(MariaDBVisitor.asString( - expressionGenerator.getRandomExpression() - )); + sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); } } @@ -79,13 +75,9 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" RETURNING "); if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString( - MariaDBExpressionGenerator.getRandomConstant(r) - )); + sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); } else { - sb.append(MariaDBVisitor.asString( - expressionGenerator.getRandomExpression() - )); + sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); } } From 18ab56017c46be4ea0309b57ce990f176ac72dbe Mon Sep 17 00:00:00 2001 From: Mohab-Sobhy Date: Mon, 3 Nov 2025 01:11:36 +0200 Subject: [PATCH 724/856] Fix Checkstyle violations and add expected errors for invalid regex in MariaDBDeleteGenerator --- src/sqlancer/mariadb/MariaDBProvider.java | 9 ++++++++- .../mariadb/gen/MariaDBDeleteGenerator.java | 15 +++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index 8f7eb78ad..c8037c019 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -19,7 +19,14 @@ import sqlancer.common.DBMSCommon; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBProvider.MariaDBGlobalState; -import sqlancer.mariadb.gen.*; +import sqlancer.mariadb.gen.MariaDBDeleteGenerator; +import sqlancer.mariadb.gen.MariaDBIndexGenerator; +import sqlancer.mariadb.gen.MariaDBInsertGenerator; +import sqlancer.mariadb.gen.MariaDBSetGenerator; +import sqlancer.mariadb.gen.MariaDBTableAdminCommandGenerator; +import sqlancer.mariadb.gen.MariaDBTableGenerator; +import sqlancer.mariadb.gen.MariaDBTruncateGenerator; +import sqlancer.mariadb.gen.MariaDBUpdateGenerator; @AutoService(DatabaseProvider.class) public class MariaDBProvider extends SQLProviderAdapter { diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index 28136eb08..6d85eb891 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -1,16 +1,16 @@ package sqlancer.mariadb.gen; +import java.util.Collections; + import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTables; import sqlancer.mariadb.MariaDBSchema; -import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; +import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.ast.MariaDBVisitor; -import java.util.Collections; - public final class MariaDBDeleteGenerator { private MariaDBDeleteGenerator() { @@ -81,6 +81,13 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { } } - return new SQLQueryAdapter(sb.toString(), errors); + String query = sb.toString(); + if (query.contains("RLIKE") || query.contains("REGEXP")) { + errors.add("Regex error"); + errors.add("quantifier does not follow a repeatable item"); + errors.add("Got error"); + } + + return new SQLQueryAdapter(query, errors); } } From 35105ec31d873b0309e86b6563a7f02ff00e409e Mon Sep 17 00:00:00 2001 From: Mohab-Sobhy Date: Thu, 20 Nov 2025 08:43:50 +0200 Subject: [PATCH 725/856] Modify DELETE generation probability --- src/sqlancer/mariadb/MariaDBProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mariadb/MariaDBProvider.java b/src/sqlancer/mariadb/MariaDBProvider.java index c8037c019..a9737f549 100644 --- a/src/sqlancer/mariadb/MariaDBProvider.java +++ b/src/sqlancer/mariadb/MariaDBProvider.java @@ -80,7 +80,7 @@ public void generateDatabase(MariaDBGlobalState globalState) throws Exception { nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case DELETE: - nrPerformed = 10; + nrPerformed = globalState.getRandomly().getInteger(0, 2); break; case SET: nrPerformed = 20; From d84210022ea9d80f8060e8d05b38d68555485b94 Mon Sep 17 00:00:00 2001 From: Pratyksh Gupta Date: Sat, 6 Dec 2025 12:38:59 +0530 Subject: [PATCH 726/856] Fix #1249: Correct REINDEX INDEX syntax to use single index instead of concatenating all --- src/sqlancer/postgres/gen/PostgresReindexGenerator.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index 9bb5ec5cd..dc0d2cf34 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -1,7 +1,6 @@ package sqlancer.postgres.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -39,7 +38,7 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { if (indexes.isEmpty()) { throw new IgnoreMeException(); } - sb.append(indexes.stream().map(i -> i.getIndexName()).collect(Collectors.joining())); + sb.append(Randomly.fromList(indexes).getIndexName()); break; case TABLE: sb.append("TABLE "); From 05bb28ce7a6f8e1152293fc8d080a94f9e4dc480 Mon Sep 17 00:00:00 2001 From: Aman Date: Sun, 7 Dec 2025 12:33:03 +0530 Subject: [PATCH 727/856] Add Spark support for TLP Oracle --- .github/workflows/main.yml | 38 ++ pom.xml | 9 +- src/sqlancer/Main.java | 2 + src/sqlancer/spark/SparkErrors.java | 67 ++++ src/sqlancer/spark/SparkGlobalState.java | 11 + src/sqlancer/spark/SparkOptions.java | 43 +++ src/sqlancer/spark/SparkProvider.java | 122 +++++++ src/sqlancer/spark/SparkSchema.java | 114 ++++++ src/sqlancer/spark/SparkToStringVisitor.java | 120 +++++++ .../spark/ast/SparkBetweenOperation.java | 10 + .../spark/ast/SparkBinaryOperation.java | 11 + .../spark/ast/SparkCaseOperation.java | 13 + .../spark/ast/SparkCastOperation.java | 25 ++ .../spark/ast/SparkColumnReference.java | 11 + src/sqlancer/spark/ast/SparkConstant.java | 194 ++++++++++ src/sqlancer/spark/ast/SparkExpression.java | 7 + src/sqlancer/spark/ast/SparkFunction.java | 13 + src/sqlancer/spark/ast/SparkInOperation.java | 12 + src/sqlancer/spark/ast/SparkJoin.java | 46 +++ src/sqlancer/spark/ast/SparkOrderingTerm.java | 10 + src/sqlancer/spark/ast/SparkSelect.java | 42 +++ .../spark/ast/SparkTableReference.java | 13 + .../spark/ast/SparkUnaryPostfixOperation.java | 13 + .../spark/ast/SparkUnaryPrefixOperation.java | 12 + .../spark/gen/SparkExpressionGenerator.java | 336 ++++++++++++++++++ .../spark/gen/SparkInsertGenerator.java | 47 +++ .../spark/gen/SparkTableGenerator.java | 100 ++++++ test/sqlancer/dbms/TestConfig.java | 1 + test/sqlancer/dbms/TestSparkTLP.java | 20 ++ 29 files changed, 1460 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/spark/SparkErrors.java create mode 100644 src/sqlancer/spark/SparkGlobalState.java create mode 100644 src/sqlancer/spark/SparkOptions.java create mode 100644 src/sqlancer/spark/SparkProvider.java create mode 100644 src/sqlancer/spark/SparkSchema.java create mode 100644 src/sqlancer/spark/SparkToStringVisitor.java create mode 100644 src/sqlancer/spark/ast/SparkBetweenOperation.java create mode 100644 src/sqlancer/spark/ast/SparkBinaryOperation.java create mode 100644 src/sqlancer/spark/ast/SparkCaseOperation.java create mode 100644 src/sqlancer/spark/ast/SparkCastOperation.java create mode 100644 src/sqlancer/spark/ast/SparkColumnReference.java create mode 100644 src/sqlancer/spark/ast/SparkConstant.java create mode 100644 src/sqlancer/spark/ast/SparkExpression.java create mode 100644 src/sqlancer/spark/ast/SparkFunction.java create mode 100644 src/sqlancer/spark/ast/SparkInOperation.java create mode 100644 src/sqlancer/spark/ast/SparkJoin.java create mode 100644 src/sqlancer/spark/ast/SparkOrderingTerm.java create mode 100644 src/sqlancer/spark/ast/SparkSelect.java create mode 100644 src/sqlancer/spark/ast/SparkTableReference.java create mode 100644 src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java create mode 100644 src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java create mode 100644 src/sqlancer/spark/gen/SparkExpressionGenerator.java create mode 100644 src/sqlancer/spark/gen/SparkInsertGenerator.java create mode 100644 src/sqlancer/spark/gen/SparkTableGenerator.java create mode 100644 test/sqlancer/dbms/TestSparkTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c53192aa..41bd92d3c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -333,6 +333,44 @@ jobs: - name: Run Tests run: HIVE_AVAILABLE=true mvn -Dtest=TestHiveTLP test + spark: + name: DBMS Tests (Spark) + runs-on: ubuntu-latest + + services: + spark: + image: apache/spark:3.5.1 + ports: + - 10000:10000 + + command: >- + /opt/spark/bin/spark-submit + --class org.apache.spark.sql.hive.thriftserver.HiveThriftServer2 + --name "Thrift JDBC/ODBC Server" + --master local[*] + --driver-memory 4g + --conf spark.hive.server2.thrift.port=10000 + --conf spark.sql.warehouse.dir=/tmp/spark-warehouse + spark-internal + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up JDK 11 + uses: actions/setup-java@v3 + with: + distribution: 'temurin' + java-version: '11' + cache: 'maven' + + - name: Build SQLancer + run: mvn -B package -DskipTests=true + + - name: Run Tests + run: SPARK_AVAILABLE=true mvn -Dtest=TestSparkTLP test + hsqldb: name: DBMS Tests (HSQLB) runs-on: ubuntu-latest diff --git a/pom.xml b/pom.xml index 2037b71ce..c7a38c9aa 100644 --- a/pom.xml +++ b/pom.xml @@ -329,7 +329,7 @@ org.slf4j - slf4j-simple + slf4j-simple 2.0.6 @@ -381,7 +381,7 @@ org.apache.hive hive-jdbc - 4.0.1 + 3.1.2 org.apache.hive @@ -393,6 +393,11 @@ hive-cli 4.0.1 + + org.apache.hadoop + hadoop-common + 3.2.4 + diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 1f2642f95..f778bd7da 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -48,6 +48,7 @@ import sqlancer.tidb.TiDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; import sqlancer.yugabyte.ysql.YSQLProvider; +import sqlancer.spark.SparkProvider; public final class Main { @@ -756,6 +757,7 @@ private static void checkForIssue799(List> providers) providers.add(new DuckDBProvider()); providers.add(new H2Provider()); providers.add(new HiveProvider()); + providers.add(new SparkProvider()); providers.add(new HSQLDBProvider()); providers.add(new MariaDBProvider()); providers.add(new MaterializeProvider()); diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java new file mode 100644 index 000000000..97c8056a3 --- /dev/null +++ b/src/sqlancer/spark/SparkErrors.java @@ -0,0 +1,67 @@ +package sqlancer.spark; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.common.query.ExpectedErrors; + +public final class SparkErrors { + + private SparkErrors() { + } + + public static List getExpressionErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("cannot resolve"); + errors.add("AnalysisException"); + errors.add("data type mismatch"); + errors.add("undefined function"); + errors.add("mismatched input"); + errors.add("due to data type mismatch"); + + // --- Invalid Literals + errors.add("The value of the typed literal"); + + errors.add("DATATYPE_MISMATCH"); + errors.add("cannot be cast to"); + + errors.add("Overflow"); + errors.add("Divide by zero"); // Common if spark.sql.ansi.enabled is true + errors.add("division by zero"); + + // --- Group By / Aggregation errors --- + errors.add("grouping expressions"); + errors.add("expression is neither present in the group by"); + errors.add("is not a valid grouping expression"); + errors.add("is not contained in either an aggregate function or the GROUP BY clause"); + errors.add("PARSE_SYNTAX_ERROR"); + errors.add("Syntax error"); + + return errors; + } + + public static void addExpressionErrors(ExpectedErrors errors) { + errors.addAll(getExpressionErrors()); + } + + public static List getInsertErrors() { + ArrayList errors = new ArrayList<>(); + + errors.add("not enough data columns"); + errors.add("cannot write to"); + errors.add("incompatible types"); + errors.add("too many data columns"); + errors.add("cannot be cast to"); + errors.add("Error running query"); + errors.add("The value of the typed literal"); + errors.add("Cannot safely cast"); // Found in logs: Decimal -> Date + errors.add("AnalysisException"); // Spark throws this for almost all insert failures + + return errors; + } + + public static void addInsertErrors(ExpectedErrors errors) { + errors.addAll(getInsertErrors()); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkGlobalState.java b/src/sqlancer/spark/SparkGlobalState.java new file mode 100644 index 000000000..e79826332 --- /dev/null +++ b/src/sqlancer/spark/SparkGlobalState.java @@ -0,0 +1,11 @@ +package sqlancer.spark; + +import sqlancer.SQLGlobalState; + +public class SparkGlobalState extends SQLGlobalState { + + @Override + protected SparkSchema readSchema() throws Exception { + return SparkSchema.fromConnection(getConnection(), getDatabaseName()); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkOptions.java b/src/sqlancer/spark/SparkOptions.java new file mode 100644 index 000000000..c9422a910 --- /dev/null +++ b/src/sqlancer/spark/SparkOptions.java @@ -0,0 +1,43 @@ +package sqlancer.spark; + +import java.sql.SQLException; +import java.util.Arrays; +import java.util.List; + +import com.beust.jcommander.Parameter; +import com.beust.jcommander.Parameters; + +import sqlancer.DBMSSpecificOptions; +import sqlancer.OracleFactory; +import sqlancer.common.oracle.TLPWhereOracle; +import sqlancer.common.oracle.TestOracle; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.spark.gen.SparkExpressionGenerator; + +@Parameters(separators = "=", commandDescription = "Spark SQL (default port: " + SparkOptions.DEFAULT_PORT + + ", default host: " + SparkOptions.DEFAULT_HOST + ")") +public class SparkOptions implements DBMSSpecificOptions { + public static final String DEFAULT_HOST = "localhost"; + public static final int DEFAULT_PORT = 10000; + + @Parameter(names = "--oracle") + public List oracle = Arrays.asList(SparkOracleFactory.TLPWhere); + + public enum SparkOracleFactory implements OracleFactory { + TLPWhere { + @Override + public TestOracle create(SparkGlobalState globalState) throws SQLException { + SparkExpressionGenerator gen = new SparkExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(SparkErrors.getExpressionErrors()) + .build(); + + return new TLPWhereOracle<>(globalState, gen, expectedErrors); + } + }; + } + + @Override + public List getTestOracleFactory() { + return oracle; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java new file mode 100644 index 000000000..f53ca10a8 --- /dev/null +++ b/src/sqlancer/spark/SparkProvider.java @@ -0,0 +1,122 @@ +package sqlancer.spark; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; + +import com.google.auto.service.AutoService; + +import sqlancer.AbstractAction; +import sqlancer.DatabaseProvider; +import sqlancer.IgnoreMeException; +import sqlancer.MainOptions; +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.SQLProviderAdapter; +import sqlancer.StatementExecutor; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLQueryProvider; +import sqlancer.spark.gen.SparkInsertGenerator; +import sqlancer.spark.gen.SparkTableGenerator; + +@AutoService(DatabaseProvider.class) +public class SparkProvider extends SQLProviderAdapter { + + public SparkProvider() { + super(SparkGlobalState.class, SparkOptions.class); + } + + public enum Action implements AbstractAction { + INSERT(SparkInsertGenerator::getQuery); // You will need to create this class + + private final SQLQueryProvider sqlQueryProvider; + + Action(SQLQueryProvider sqlQueryProvider) { + this.sqlQueryProvider = sqlQueryProvider; + } + + @Override + public SQLQueryAdapter getQuery(SparkGlobalState state) throws Exception { + return sqlQueryProvider.getQuery(state); + } + } + + private static int mapActions(SparkGlobalState globalState, Action a) { + Randomly r = globalState.getRandomly(); + switch (a) { + case INSERT: + return r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); + default: + throw new AssertionError(a); + } + } + + @Override + public void generateDatabase(SparkGlobalState globalState) throws Exception { + for (int i = 0; i < Randomly.fromOptions(1, 2); i++) { + boolean success; + do { + String tableName = globalState.getSchema().getFreeTableName(); + SQLQueryAdapter qt = SparkTableGenerator.generate(globalState, tableName); + success = globalState.executeStatement(qt); + } while (!success); + } + + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + + StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), + SparkProvider::mapActions, (q) -> { + if (globalState.getSchema().getDatabaseTables().isEmpty()) { + throw new IgnoreMeException(); + } + }); + se.executeStatements(); + } + + @Override + public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLException { + String username = globalState.getOptions().getUserName(); + String password = globalState.getOptions().getPassword(); + String host = globalState.getOptions().getHost(); + int port = globalState.getOptions().getPort(); + + if (host == null) { + host = SparkOptions.DEFAULT_HOST; + } + if (port == MainOptions.NO_SET_PORT) { + port = SparkOptions.DEFAULT_PORT; + } + + String databaseName = globalState.getDatabaseName(); + + // Spark uses the Hive driver for JDBC usually + String url = String.format("jdbc:hive2://%s:%d/%s", host, port, "default"); + + // Connect to default to create the fuzzing DB + Connection con = DriverManager.getConnection(url, username, password); + try (Statement s = con.createStatement()) { + s.execute("DROP DATABASE IF EXISTS " + databaseName + " CASCADE"); + } + try (Statement s = con.createStatement()) { + s.execute("CREATE DATABASE " + databaseName); + } + con.close(); + + // Connect to the specific fuzzing DB + con = DriverManager.getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName), username, + password); + try (Statement s = con.createStatement()) { + // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the generator loves to do. + s.execute("SET spark.sql.ansi.enabled=false"); + } + return new SQLConnection(con); + } + + @Override + public String getDBMSName() { + return "spark"; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java new file mode 100644 index 000000000..849652b19 --- /dev/null +++ b/src/sqlancer/spark/SparkSchema.java @@ -0,0 +1,114 @@ +package sqlancer.spark; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.SQLConnection; +import sqlancer.common.schema.AbstractRelationalTable; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; +import sqlancer.common.schema.TableIndex; +import sqlancer.spark.SparkSchema.SparkTable; + +public class SparkSchema extends AbstractSchema { + + public enum SparkDataType { + STRING, INTEGER, DOUBLE, BOOLEAN, TIMESTAMP, DATE; + + public static SparkDataType getRandomType() { + return Randomly.fromList(Arrays.asList(values())); + } + } + + public static class SparkColumn extends AbstractTableColumn { + public SparkColumn(String name, SparkTable table, SparkDataType type) { + super(name, table, type); + } + } + + public static class SparkTables extends AbstractTables { + public SparkTables(List tables) { + super(tables); + } + } + + public static class SparkTable extends AbstractRelationalTable { + public SparkTable(String name, List columns, boolean isView) { + super(name, columns, Collections.emptyList(), isView); + } + } + + public SparkSchema(List databaseTables) { + super(databaseTables); + } + + public static SparkSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { + List databaseTables = new ArrayList<>(); + List tableNames = getTableNames(con); + for (String tableName : tableNames) { + List databaseColumns = getTableColumns(con, tableName); + boolean isView = tableName.toLowerCase().startsWith("v"); + SparkTable t = new SparkTable(tableName, databaseColumns, isView); + for (SparkColumn c : databaseColumns) { + c.setTable(t); + } + databaseTables.add(t); + } + return new SparkSchema(databaseTables); + } + + private static List getTableNames(SQLConnection con) throws SQLException { + List tableNames = new ArrayList<>(); + try (Statement s = con.createStatement()) { + ResultSet tableRs = s.executeQuery("SHOW TABLES"); + while (tableRs.next()) { + // Spark SHOW TABLES output: database, tableName, isTemporary + String tableName = tableRs.getString("tableName"); + tableNames.add(tableName); + } + } + return tableNames; + } + + private static List getTableColumns(SQLConnection con, String tableName) throws SQLException { + List columns = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format("DESCRIBE %s", tableName))) { + while (rs.next()) { + String columnName = rs.getString("col_name"); + String dataType = rs.getString("data_type"); + // Filter out Spark partition info or comments usually at bottom of describe + if (columnName.startsWith("#") || columnName.isEmpty()) + continue; + + columns.add(new SparkColumn(columnName, null, getColumnType(dataType))); + } + } + } + return columns; + } + + private static SparkDataType getColumnType(String typeString) { + String upper = typeString.toUpperCase(); + if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) + return SparkDataType.STRING; + if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) + return SparkDataType.INTEGER; + if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) + return SparkDataType.DOUBLE; + if (upper.startsWith("BOOLEAN")) + return SparkDataType.BOOLEAN; + if (upper.startsWith("TIMESTAMP")) + return SparkDataType.TIMESTAMP; + if (upper.startsWith("DATE")) + return SparkDataType.DATE; + return SparkDataType.STRING; // Fallback + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/SparkToStringVisitor.java b/src/sqlancer/spark/SparkToStringVisitor.java new file mode 100644 index 000000000..91f47e32c --- /dev/null +++ b/src/sqlancer/spark/SparkToStringVisitor.java @@ -0,0 +1,120 @@ +package sqlancer.spark; + +import sqlancer.common.ast.newast.NewToStringVisitor; +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.spark.ast.SparkCastOperation; +import sqlancer.spark.ast.SparkConstant; +import sqlancer.spark.ast.SparkExpression; +import sqlancer.spark.ast.SparkJoin; +import sqlancer.spark.ast.SparkSelect; + +public class SparkToStringVisitor extends NewToStringVisitor { + + @Override + public void visitSpecific(SparkExpression expr) { + if (expr instanceof SparkConstant) { + visit((SparkConstant) expr); + } else if (expr instanceof SparkSelect) { + visit((SparkSelect) expr); + } else if (expr instanceof SparkJoin) { + visit((SparkJoin) expr); + } else if (expr instanceof SparkCastOperation) { + visit((SparkCastOperation) expr); + } else { + throw new AssertionError(expr.getClass()); + } + } + + private void visit(SparkConstant constant) { + sb.append(constant.toString()); + } + + private void visit(SparkSelect select) { + sb.append("SELECT "); + if (select.isDistinct()) { + sb.append("DISTINCT "); + } + visit(select.getFetchColumns()); + sb.append(" FROM "); + visit(select.getFromList()); + if (!select.getFromList().isEmpty() && !select.getJoinList().isEmpty()) { + sb.append(", "); + } + if (!select.getJoinList().isEmpty()) { + visit(select.getJoinList()); + } + if (select.getWhereClause() != null) { + sb.append(" WHERE "); + visit(select.getWhereClause()); + } + if (!select.getGroupByExpressions().isEmpty()) { + sb.append(" GROUP BY "); + visit(select.getGroupByExpressions()); + } + if (select.getHavingClause() != null) { + sb.append(" HAVING "); + visit(select.getHavingClause()); + } + if (!select.getOrderByClauses().isEmpty()) { + sb.append(" ORDER BY "); + visit(select.getOrderByClauses()); + } + if (select.getLimitClause() != null) { + sb.append(" LIMIT "); + visit(select.getLimitClause()); + } + // Spark supports OFFSET, though strictly usually with LIMIT or in newer versions + if (select.getOffsetClause() != null) { + sb.append(" OFFSET "); + visit(select.getOffsetClause()); + } + } + + private void visit(SparkJoin join) { + switch (join.getJoinType()) { + case INNER: + sb.append(" INNER JOIN "); + break; + case LEFT_OUTER: + sb.append(" LEFT JOIN "); + break; + case RIGHT_OUTER: + sb.append(" RIGHT JOIN "); + break; + case FULL_OUTER: + sb.append(" FULL JOIN "); + break; + case LEFT_SEMI: + sb.append(" LEFT SEMI JOIN "); + break; + // Spark also supports LEFT ANTI, which Hive might lack in some older versions + case LEFT_ANTI: + sb.append(" LEFT ANTI JOIN "); + break; + case CROSS: + sb.append(" CROSS JOIN "); + break; + default: + throw new UnsupportedOperationException("Join type not supported in Spark visitor: " + join.getJoinType()); + } + visit((TableReferenceNode) join.getRightTable()); + if (join.getOnClause() != null) { + sb.append(" ON "); + visit(join.getOnClause()); + } + } + + private void visit(SparkCastOperation cast) { + sb.append("CAST("); + visit(cast.getExpression()); + sb.append(" AS "); + sb.append(cast.getType()); + sb.append(")"); + } + + public static String asString(SparkExpression expr) { + SparkToStringVisitor visitor = new SparkToStringVisitor(); + visitor.visit(expr); + return visitor.get(); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkBetweenOperation.java b/src/sqlancer/spark/ast/SparkBetweenOperation.java new file mode 100644 index 000000000..f229c1c7c --- /dev/null +++ b/src/sqlancer/spark/ast/SparkBetweenOperation.java @@ -0,0 +1,10 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.NewBetweenOperatorNode; + +public class SparkBetweenOperation extends NewBetweenOperatorNode implements SparkExpression { + + public SparkBetweenOperation(SparkExpression left, SparkExpression middle, SparkExpression right, boolean isTrue) { + super(left, middle, right, isTrue); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkBinaryOperation.java b/src/sqlancer/spark/ast/SparkBinaryOperation.java new file mode 100644 index 000000000..04af0ec4c --- /dev/null +++ b/src/sqlancer/spark/ast/SparkBinaryOperation.java @@ -0,0 +1,11 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewBinaryOperatorNode; + +public class SparkBinaryOperation extends NewBinaryOperatorNode implements SparkExpression { + + public SparkBinaryOperation(SparkExpression left, SparkExpression right, Operator op) { + super(left, right, op); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkCaseOperation.java b/src/sqlancer/spark/ast/SparkCaseOperation.java new file mode 100644 index 000000000..fb1ee0cd8 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkCaseOperation.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewCaseOperatorNode; + +public class SparkCaseOperation extends NewCaseOperatorNode implements SparkExpression { + + public SparkCaseOperation(SparkExpression switchCondition, List conditions, + List expressions, SparkExpression elseExpr) { + super(switchCondition, conditions, expressions, elseExpr); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkCastOperation.java b/src/sqlancer/spark/ast/SparkCastOperation.java new file mode 100644 index 000000000..3bc5eb30d --- /dev/null +++ b/src/sqlancer/spark/ast/SparkCastOperation.java @@ -0,0 +1,25 @@ +package sqlancer.spark.ast; + +import sqlancer.spark.SparkSchema.SparkDataType; + +public class SparkCastOperation implements SparkExpression { + + private final SparkExpression expression; + private final SparkDataType type; + + public SparkCastOperation(SparkExpression expression, SparkDataType type) { + if (expression == null) { + throw new AssertionError(); + } + this.expression = expression; + this.type = type; + } + + public SparkExpression getExpression() { + return expression; + } + + public SparkDataType getType() { + return type; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkColumnReference.java b/src/sqlancer/spark/ast/SparkColumnReference.java new file mode 100644 index 000000000..75e92d267 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkColumnReference.java @@ -0,0 +1,11 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.ColumnReferenceNode; +import sqlancer.spark.SparkSchema.SparkColumn; + +public class SparkColumnReference extends ColumnReferenceNode implements SparkExpression { + + public SparkColumnReference(SparkColumn column) { + super(column); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkConstant.java b/src/sqlancer/spark/ast/SparkConstant.java new file mode 100644 index 000000000..9f73af59f --- /dev/null +++ b/src/sqlancer/spark/ast/SparkConstant.java @@ -0,0 +1,194 @@ +package sqlancer.spark.ast; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +public abstract class SparkConstant implements SparkExpression { + + public boolean isNull() { + return false; + } + + public static class SparkNullConstant extends SparkConstant { + + @Override + public boolean isNull() { + return true; + } + + @Override + public String toString() { + return "NULL"; + } + } + + public static class SparkIntConstant extends SparkConstant { + + private final long value; + + public SparkIntConstant(long value) { + this.value = value; + } + + public long getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class SparkDoubleConstant extends SparkConstant { + + private final double value; + + public SparkDoubleConstant(double value) { + this.value = value; + } + + public double getValue() { + return value; + } + + @Override + public String toString() { + if (value == Double.POSITIVE_INFINITY) { + return "CAST('Infinity' AS DOUBLE)"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "CAST('-Infinity' AS DOUBLE)"; + } else if (Double.isNaN(value)) { + return "CAST('NaN' AS DOUBLE)"; + } + return String.valueOf(value); + } + } + + public static class SparkDecimalConstant extends SparkConstant { + + private final BigDecimal value; + + public SparkDecimalConstant(BigDecimal value) { + this.value = value; + } + + public BigDecimal getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static class SparkTimestampConstant extends SparkConstant { + + private final String textRepr; + + public SparkTimestampConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); // Spark prefers full timestamp + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("TIMESTAMP '%s'", textRepr); + } + } + + public static class SparkDateConstant extends SparkConstant { + + private final String textRepr; + + public SparkDateConstant(long value) { + Timestamp timestamp = new Timestamp(value); + SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd"); + this.textRepr = dateFormat.format(timestamp); + } + + public String getValue() { + return textRepr; + } + + @Override + public String toString() { + return String.format("DATE '%s'", textRepr); + } + } + + public static class SparkStringConstant extends SparkConstant { + + private final String value; + + public SparkStringConstant(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + + @Override + public String toString() { + return "'" + value.replace("'", "''").replace("\\", "\\\\") + "'"; + } + } + + public static class SparkBooleanConstant extends SparkConstant { + + private final boolean value; + + public SparkBooleanConstant(boolean value) { + this.value = value; + } + + public boolean getValue() { + return value; + } + + @Override + public String toString() { + return String.valueOf(value); + } + } + + public static SparkConstant createNullConstant() { + return new SparkNullConstant(); + } + + public static SparkConstant createIntConstant(long value) { + return new SparkIntConstant(value); + } + + public static SparkConstant createDoubleConstant(double value) { + return new SparkDoubleConstant(value); + } + + public static SparkConstant createDecimalConstant(BigDecimal value) { + return new SparkDecimalConstant(value); + } + + public static SparkConstant createTimestampConstant(long value) { + return new SparkTimestampConstant(value); + } + + public static SparkConstant createDateConstant(long value) { + return new SparkDateConstant(value); + } + + public static SparkConstant createStringConstant(String value) { + return new SparkStringConstant(value); + } + + public static SparkConstant createBooleanConstant(boolean value) { + return new SparkBooleanConstant(value); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkExpression.java b/src/sqlancer/spark/ast/SparkExpression.java new file mode 100644 index 000000000..a130096e3 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkExpression.java @@ -0,0 +1,7 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.spark.SparkSchema.SparkColumn; + +public interface SparkExpression extends Expression { +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkFunction.java b/src/sqlancer/spark/ast/SparkFunction.java new file mode 100644 index 000000000..d5740ee36 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkFunction.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewFunctionNode; + +public class SparkFunction extends NewFunctionNode implements SparkExpression { + + public SparkFunction(List args, F func) { + super(args, func); + } + +} diff --git a/src/sqlancer/spark/ast/SparkInOperation.java b/src/sqlancer/spark/ast/SparkInOperation.java new file mode 100644 index 000000000..37a80e3ff --- /dev/null +++ b/src/sqlancer/spark/ast/SparkInOperation.java @@ -0,0 +1,12 @@ +package sqlancer.spark.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewInOperatorNode; + +public class SparkInOperation extends NewInOperatorNode implements SparkExpression { + + public SparkInOperation(SparkExpression left, List right, boolean isNegated) { + super(left, right, isNegated); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkJoin.java b/src/sqlancer/spark/ast/SparkJoin.java new file mode 100644 index 000000000..44da7fba4 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkJoin.java @@ -0,0 +1,46 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.Join; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; + +public class SparkJoin implements SparkExpression, Join { + + private final SparkTableReference leftTable; + private final SparkTableReference rightTable; + private final JoinType joinType; + private SparkExpression onClause; + + public enum JoinType { + INNER, LEFT_OUTER, RIGHT_OUTER, FULL_OUTER, LEFT_SEMI, LEFT_ANTI, CROSS; + } + + public SparkJoin(SparkTableReference leftTable, SparkTableReference rightTable, JoinType joinType, + SparkExpression onClause) { + this.leftTable = leftTable; + this.rightTable = rightTable; + this.joinType = joinType; + this.onClause = onClause; + } + + public SparkTableReference getLeftTable() { + return leftTable; + } + + public SparkTableReference getRightTable() { + return rightTable; + } + + public JoinType getJoinType() { + return joinType; + } + + public SparkExpression getOnClause() { + return onClause; + } + + @Override + public void setOnClause(SparkExpression onClause) { + this.onClause = onClause; + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkOrderingTerm.java b/src/sqlancer/spark/ast/SparkOrderingTerm.java new file mode 100644 index 000000000..824801c00 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkOrderingTerm.java @@ -0,0 +1,10 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.NewOrderingTerm; + +public class SparkOrderingTerm extends NewOrderingTerm implements SparkExpression { + + public SparkOrderingTerm(SparkExpression expr, Ordering ordering) { + super(expr, ordering); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkSelect.java b/src/sqlancer/spark/ast/SparkSelect.java new file mode 100644 index 000000000..0986ce0a6 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkSelect.java @@ -0,0 +1,42 @@ +package sqlancer.spark.ast; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.common.ast.SelectBase; +import sqlancer.common.ast.newast.Select; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkSelect extends SelectBase + implements Select, SparkExpression { + + private boolean isDistinct; + + public void setDistinct(boolean isDistinct) { + this.isDistinct = isDistinct; + } + + public boolean isDistinct() { + return isDistinct; + } + + @Override + public void setJoinClauses(List joinStatements) { + List expressions = joinStatements.stream().map(e -> (SparkExpression) e) + .collect(Collectors.toList()); + setJoinList(expressions); + } + + @Override + public List getJoinClauses() { + return getJoinList().stream().map(e -> (SparkJoin) e).collect(Collectors.toList()); + } + + @Override + public String asString() { + return SparkToStringVisitor.asString(this); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkTableReference.java b/src/sqlancer/spark/ast/SparkTableReference.java new file mode 100644 index 000000000..92a59ad3d --- /dev/null +++ b/src/sqlancer/spark/ast/SparkTableReference.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.spark.SparkSchema; + +public class SparkTableReference extends TableReferenceNode + implements SparkExpression { + + public SparkTableReference(SparkSchema.SparkTable table) { + super(table); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java new file mode 100644 index 000000000..f1082a655 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java @@ -0,0 +1,13 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPostfixOperatorNode; + +public class SparkUnaryPostfixOperation extends NewUnaryPostfixOperatorNode + implements SparkExpression { + + public SparkUnaryPostfixOperation(SparkExpression expr, Operator op) { + super(expr, op); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java new file mode 100644 index 000000000..d1bd94ab4 --- /dev/null +++ b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java @@ -0,0 +1,12 @@ +package sqlancer.spark.ast; + +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewUnaryPrefixOperatorNode; + +public class SparkUnaryPrefixOperation extends NewUnaryPrefixOperatorNode implements SparkExpression { + + public SparkUnaryPrefixOperation(SparkExpression expr, Operator op) { + super(expr, op); + } + +} \ No newline at end of file diff --git a/src/sqlancer/spark/gen/SparkExpressionGenerator.java b/src/sqlancer/spark/gen/SparkExpressionGenerator.java new file mode 100644 index 000000000..faf8a07f0 --- /dev/null +++ b/src/sqlancer/spark/gen/SparkExpressionGenerator.java @@ -0,0 +1,336 @@ +package sqlancer.spark.gen; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.gen.TLPWhereGenerator; +import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.schema.AbstractTables; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkDataType; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.ast.SparkBetweenOperation; +import sqlancer.spark.ast.SparkBinaryOperation; +import sqlancer.spark.ast.SparkCaseOperation; +import sqlancer.spark.ast.SparkCastOperation; +import sqlancer.spark.ast.SparkColumnReference; +import sqlancer.spark.ast.SparkConstant; +import sqlancer.spark.ast.SparkExpression; +import sqlancer.spark.ast.SparkFunction; +import sqlancer.spark.ast.SparkInOperation; +import sqlancer.spark.ast.SparkJoin; +import sqlancer.spark.ast.SparkOrderingTerm; +import sqlancer.spark.ast.SparkSelect; +import sqlancer.spark.ast.SparkTableReference; +import sqlancer.spark.ast.SparkUnaryPostfixOperation; +import sqlancer.spark.ast.SparkUnaryPrefixOperation; + +public class SparkExpressionGenerator extends UntypedExpressionGenerator + implements TLPWhereGenerator { + + private final SparkGlobalState globalState; + private List tables; + + private enum Expression { + UNARY_PREFIX, UNARY_POSTFIX, BINARY_COMPARISON, BINARY_LOGICAL, BINARY_ARITHMETIC, CAST, FUNC, BETWEEN, IN, + CASE; + } + + public SparkExpressionGenerator(SparkGlobalState globalState) { + this.globalState = globalState; + } + + @Override + public SparkExpression negatePredicate(SparkExpression predicate) { + return new SparkUnaryPrefixOperation(predicate, SparkUnaryPrefixOperator.NOT); + } + + @Override + public SparkExpression isNull(SparkExpression expr) { + return new SparkUnaryPostfixOperation(expr, SparkUnaryPostfixOperator.IS_NULL); + } + + @Override + protected SparkExpression generateExpression(int depth) { + return generateExpressionInternal(depth); + } + + private SparkExpression generateExpressionInternal(int depth) throws AssertionError { + if (depth >= globalState.getOptions().getMaxExpressionDepth() + || Randomly.getBooleanWithRatherLowProbability()) { + return generateLeafNode(); + } + if (allowAggregates && Randomly.getBooleanWithRatherLowProbability()) { + allowAggregates = false; // aggregate function calls cannot be nested + SparkAggregateFunction aggregate = SparkAggregateFunction.getRandom(); + return new SparkFunction<>(generateExpressions(aggregate.getNrArgs(), depth + 1), aggregate); + } + + List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); + Expression expr = Randomly.fromList(possibleOptions); + + switch (expr) { + case UNARY_PREFIX: + return new SparkUnaryPrefixOperation(generateExpression(depth + 1), SparkUnaryPrefixOperator.getRandom()); + case UNARY_POSTFIX: + return new SparkUnaryPostfixOperation(generateExpression(depth + 1), SparkUnaryPostfixOperator.getRandom()); + case BINARY_COMPARISON: + Operator op = SparkBinaryComparisonOperator.getRandom(); + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_LOGICAL: + op = SparkBinaryLogicalOperator.getRandom(); + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), op); + case BINARY_ARITHMETIC: + return new SparkBinaryOperation(generateExpression(depth + 1), generateExpression(depth + 1), + SparkBinaryArithmeticOperator.getRandom()); + case CAST: + return new SparkCastOperation(generateExpression(depth + 1), SparkDataType.getRandomType()); + case FUNC: + SparkFunc func = SparkFunc.getRandom(); + return new SparkFunction<>(generateExpressions(func.getNrArgs()), func); + case BETWEEN: + return new SparkBetweenOperation(generateExpression(depth + 1), generateExpression(depth + 1), + generateExpression(depth + 1), Randomly.getBoolean()); + case IN: + return new SparkInOperation(generateExpression(depth + 1), + generateExpressions(Randomly.smallNumber() + 1, depth + 1), Randomly.getBoolean()); + case CASE: + int nr = Randomly.smallNumber() + 1; + return new SparkCaseOperation(generateExpression(depth + 1), generateExpressions(nr, depth + 1), + generateExpressions(nr, depth + 1), generateExpression(depth + 1)); + default: + throw new AssertionError(expr); + } + } + + @Override + public SparkExpression generateConstant() { + if (Randomly.getBooleanWithRatherLowProbability()) { + return SparkConstant.createNullConstant(); + } + SparkDataType[] values = SparkDataType.values(); + SparkDataType constantType = Randomly.fromOptions(values); + switch (constantType) { + case STRING: + return SparkConstant.createStringConstant(globalState.getRandomly().getString()); + case INTEGER: + return SparkConstant.createIntConstant(globalState.getRandomly().getInteger()); + case DOUBLE: + return SparkConstant.createDoubleConstant(globalState.getRandomly().getDouble()); + case BOOLEAN: + return SparkConstant.createBooleanConstant(Randomly.getBoolean()); + case TIMESTAMP: + return SparkConstant.createTimestampConstant(globalState.getRandomly().getInteger()); + case DATE: + return SparkConstant.createDateConstant(globalState.getRandomly().getInteger()); + default: + throw new AssertionError(constantType); + } + } + + @Override + protected SparkExpression generateColumn() { + SparkColumn column = Randomly.fromList(columns); + return new SparkColumnReference(column); + } + + @Override + public List generateOrderBys() { + List expr = super.generateOrderBys(); + List newExpr = new ArrayList<>(expr.size()); + for (SparkExpression curExpr : expr) { + if (Randomly.getBoolean()) { + curExpr = new SparkOrderingTerm(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + } + + @Override + public SparkExpressionGenerator setTablesAndColumns(AbstractTables tables) { + this.columns = tables.getColumns(); + this.tables = tables.getTables(); + return this; + } + + @Override + public SparkExpression generateBooleanExpression() { + return generateExpression(); + } + + @Override + public SparkSelect generateSelect() { + return new SparkSelect(); + } + + @Override + public List getTableRefs() { + return tables.stream().map(t -> new SparkTableReference(t)).collect(Collectors.toList()); + } + + @Override + public List generateFetchColumns(boolean allowAggregates) { + if (Randomly.getBoolean()) { + return List.of(new SparkColumnReference(new SparkColumn("*", null, null))); + } + return Randomly.nonEmptySubset(columns).stream().map(c -> new SparkColumnReference(c)) + .collect(Collectors.toList()); + } + + @Override + public List getRandomJoinClauses() { + return List.of(); + } + + public enum SparkUnaryPrefixOperator implements Operator { + NOT("NOT"), PLUS("+"), MINUS("-"), BITWISE_NOT("~"); + + private String textRepr; + + SparkUnaryPrefixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkUnaryPrefixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkUnaryPostfixOperator implements Operator { + IS_NULL("IS NULL"), IS_NOT_NULL("IS NOT NULL"); + + private String textRepr; + + SparkUnaryPostfixOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkUnaryPostfixOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryComparisonOperator implements Operator { + EQUALS("="), GREATER(">"), GREATER_EQUALS(">="), SMALLER("<"), SMALLER_EQUALS("<="), NOT_EQUALS("!="), + LIKE("LIKE"), NOT_LIKE("NOT LIKE"), RLIKE("RLIKE"); + + private String textRepr; + + SparkBinaryComparisonOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryComparisonOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryLogicalOperator implements Operator { + AND("AND"), OR("OR"); + + private String textRepr; + + SparkBinaryLogicalOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryLogicalOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkBinaryArithmeticOperator implements Operator { + // Spark supports || for concat, and bitwise operators &, |, ^ + CONCAT("||"), ADD("+"), SUB("-"), MULT("*"), DIV("/"), MOD("%"), BITWISE_AND("&"), BITWISE_OR("|"), + BITWISE_XOR("^"); + + private String textRepr; + + SparkBinaryArithmeticOperator(String textRepr) { + this.textRepr = textRepr; + } + + public static SparkBinaryArithmeticOperator getRandom() { + return Randomly.fromOptions(values()); + } + + @Override + public String getTextRepresentation() { + return textRepr; + } + } + + public enum SparkAggregateFunction { + COUNT(1), SUM(1), AVG(1), MIN(1), MAX(1), VARIANCE(1), VAR_SAMP(1), STDDEV_POP(1), STDDEV_SAMP(1), COVAR_POP(2), + COVAR_SAMP(2), CORR(2); + + private int nrArgs; + + SparkAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static SparkAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + } + + public enum SparkFunc { + ROUND(2), FLOOR(1), ABS(1), CEIL(1); + + private int nrArgs; + private boolean isVariadic; + + SparkFunc(int nrArgs) { + this(nrArgs, false); + } + + SparkFunc(int nrArgs, boolean isVariadic) { + this.nrArgs = nrArgs; + this.isVariadic = isVariadic; + } + + public static SparkFunc getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + if (isVariadic) { + return Randomly.smallNumber() + nrArgs; + } else { + return nrArgs; + } + } + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java new file mode 100644 index 000000000..29232fdb2 --- /dev/null +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -0,0 +1,47 @@ +package sqlancer.spark.gen; + +import java.util.List; + +import sqlancer.common.gen.AbstractInsertGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.spark.SparkErrors; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkInsertGenerator extends AbstractInsertGenerator { + + private final SparkGlobalState globalState; + private final ExpectedErrors errors = new ExpectedErrors(); + private final SparkExpressionGenerator gen; + + public SparkInsertGenerator(SparkGlobalState globalState) { + this.globalState = globalState; + this.gen = new SparkExpressionGenerator(globalState); + } + + public static SQLQueryAdapter getQuery(SparkGlobalState globalState) { + return new SparkInsertGenerator(globalState).generate(); + } + + @Override + protected void insertValue(SparkColumn column) { + sb.append(SparkToStringVisitor.asString(gen.generateConstant())); + } + + private SQLQueryAdapter generate() { + sb.append("INSERT INTO "); + SparkTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append(table.getName()); + + sb.append(" VALUES "); + + List columns = table.getColumns(); + insertColumns(columns); + + SparkErrors.addInsertErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, false, false); + } +} \ No newline at end of file diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java new file mode 100644 index 000000000..68cafdafb --- /dev/null +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -0,0 +1,100 @@ +package sqlancer.spark.gen; + +import java.util.ArrayList; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.DBMSCommon; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.spark.SparkErrors; +import sqlancer.spark.SparkGlobalState; +import sqlancer.spark.SparkSchema; +import sqlancer.spark.SparkSchema.SparkColumn; +import sqlancer.spark.SparkSchema.SparkDataType; +import sqlancer.spark.SparkSchema.SparkTable; +import sqlancer.spark.SparkToStringVisitor; + +public class SparkTableGenerator { + + private enum ColumnConstraints { + NOT_NULL, DEFAULT + // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources (Parquet/ORC) + // without specific catalogs (like Delta/Iceberg), so we limit to constraints Spark SQL widely accepts. + } + + private final SparkGlobalState globalState; + private final String tableName; + private final StringBuilder sb = new StringBuilder(); + private final SparkExpressionGenerator gen; + private final SparkTable table; + private final List columnsToBeAdded = new ArrayList<>(); + + public SparkTableGenerator(SparkGlobalState globalState, String tableName) { + this.tableName = tableName; + this.globalState = globalState; + this.table = new SparkTable(tableName, columnsToBeAdded, false); + this.gen = new SparkExpressionGenerator(globalState).setColumns(columnsToBeAdded); + } + + public static SQLQueryAdapter generate(SparkGlobalState globalState, String tableName) { + SparkTableGenerator generator = new SparkTableGenerator(globalState, tableName); + return generator.create(); + } + + private SQLQueryAdapter create() { + ExpectedErrors errors = new ExpectedErrors(); + + sb.append("CREATE TABLE "); + sb.append(globalState.getDatabaseName()); + sb.append("."); + sb.append(tableName); + sb.append(" ("); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + if (i != 0) { + sb.append(", "); + } + appendColumn(i); + } + sb.append(")"); + sb.append(" USING PARQUET"); + + // TODO: implement PARTITION BY clause + // TODO: implement CLUSTERED BY clauses + // TODO: implement ROW FORMAT and STORED AS clauses + // TODO: randomly add some predefined TABLEPROPERTIES + + SparkErrors.addExpressionErrors(errors); + return new SQLQueryAdapter(sb.toString(), errors, true, false); + } + + private void appendColumn(int columnId) { + String columnName = DBMSCommon.createColumnName(columnId); + sb.append(columnName); + sb.append(" "); + SparkDataType randType = SparkSchema.SparkDataType.getRandomType(); + sb.append(randType); + columnsToBeAdded.add(new SparkColumn(columnName, table, randType)); + appendColumnConstraint(); + } + + private void appendColumnConstraint() { + if (Randomly.getBoolean()) { + return; + } + + ColumnConstraints constraint = Randomly.fromOptions(ColumnConstraints.values()); + switch (constraint) { + case NOT_NULL: + sb.append(" NOT NULL"); + break; + case DEFAULT: + sb.append(" DEFAULT "); + sb.append(SparkToStringVisitor.asString(gen.generateConstant())); + sb.append(" "); + break; + default: + throw new AssertionError(constraint); + } + } +} \ No newline at end of file diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index f5aeefa12..f6be45648 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -11,6 +11,7 @@ public class TestConfig { public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; public static final String DORIS_ENV = "DORIS_AVAILABLE"; public static final String HIVE_ENV = "HIVE_AVAILABLE"; + public static final String SPARK_ENV = "SPARK_AVAILABLE"; public static final String MARIADB_ENV = "MARIADB_AVAILABLE"; public static final String MATERIALIZE_ENV = "MATERIALIZE_AVAILABLE"; public static final String MYSQL_ENV = "MYSQL_AVAILABLE"; diff --git a/test/sqlancer/dbms/TestSparkTLP.java b/test/sqlancer/dbms/TestSparkTLP.java new file mode 100644 index 000000000..83302ceff --- /dev/null +++ b/test/sqlancer/dbms/TestSparkTLP.java @@ -0,0 +1,20 @@ +package sqlancer.dbms; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import org.junit.jupiter.api.Test; + +import sqlancer.Main; + +public class TestSparkTLP { + + @Test + public void testSparkTLPWhere() { + assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.SPARK_ENV)); + assertEquals(0, + Main.executeMain(new String[] { "--canonicalize-sql-strings", "false", "--random-seed", "0", + "--timeout-seconds", TestConfig.SECONDS, "--num-threads", "1", "--num-queries", + TestConfig.NUM_QUERIES, "spark", "--oracle", "TLPWhere" })); + } +} \ No newline at end of file From b4c3763d9db5b179e41c170dfd5d7fa31d4bc9eb Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 10:48:47 +0000 Subject: [PATCH 728/856] Fix where clause in Materialize --- src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java index d61e452f8..f7ff76305 100644 --- a/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeExpressionGenerator.java @@ -600,6 +600,7 @@ public String generateOptimizedQueryString(MaterializeSelect select, Materialize } select.setSelectType(SelectType.ALL); } + select.setWhereClause(whereCondition); return select.asString(); } From e4e013d064232ea7f9d17b8bdbb5fb2411c035ae Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 10:55:54 +0000 Subject: [PATCH 729/856] Fix up Materialize CI --- .github/workflows/main.yml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c53192aa..026fa52bc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -392,10 +392,7 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest - sleep 5 - # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 - psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -420,10 +417,7 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -d -p6875:6875 -p6877:6877 -p 26257:26257 materialize/materialized:latest - sleep 5 - # Workaround for https://github.com/cockroachdb/cockroach/issues/93892 - psql postgres://root@localhost:26257 -c "SET CLUSTER SETTING sql.stats.forecasts.enabled = false" + docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest - name: Set up JDK 11 uses: actions/setup-java@v3 with: From dacd33c3ba671bed063d618d864c14d5bba0a4f5 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 12:50:10 +0000 Subject: [PATCH 730/856] Try to fix duplicate table creation in Materialize --- src/sqlancer/materialize/MaterializeProvider.java | 11 ++++++++--- .../materialize/gen/MaterializeTableGenerator.java | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/materialize/MaterializeProvider.java b/src/sqlancer/materialize/MaterializeProvider.java index 5b18b1dfb..e7bdb4c4f 100644 --- a/src/sqlancer/materialize/MaterializeProvider.java +++ b/src/sqlancer/materialize/MaterializeProvider.java @@ -240,12 +240,17 @@ protected void readFunctions(MaterializeGlobalState globalState) throws SQLExcep } protected void createTables(MaterializeGlobalState globalState, int numTables) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < numTables) { + int existingTables = globalState.getSchema().getDatabaseTables().size(); + int createdTables = 0; + int nextTableIndex = existingTables; + while (existingTables + createdTables < numTables) { try { - String tableName = DBMSCommon.createTableName(globalState.getSchema().getDatabaseTables().size()); + String tableName = DBMSCommon.createTableName(nextTableIndex++); SQLQueryAdapter createTable = MaterializeTableGenerator.generate(tableName, globalState.getSchema(), generateOnlyKnown, globalState); - globalState.executeStatement(createTable); + if (globalState.executeStatement(createTable)) { + createdTables++; + } } catch (IgnoreMeException e) { } diff --git a/src/sqlancer/materialize/gen/MaterializeTableGenerator.java b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java index f132f7370..c6772db47 100644 --- a/src/sqlancer/materialize/gen/MaterializeTableGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeTableGenerator.java @@ -52,6 +52,7 @@ public MaterializeTableGenerator(String tableName, MaterializeSchema newSchema, errors.add("no collation was derived for partition key column"); errors.add("inherits from generated column but specifies identity"); errors.add("inherits from generated column but specifies default"); + errors.add("already exists"); MaterializeCommon.addCommonExpressionErrors(errors); MaterializeCommon.addCommonTableErrors(errors); } From 44a156e8fe7c9c913c782808ce0540beca3be342 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Feb 2026 14:10:22 +0000 Subject: [PATCH 731/856] Adapt materialize insert error message --- src/sqlancer/materialize/gen/MaterializeInsertGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java index 01015514b..7a5374b95 100644 --- a/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeInsertGenerator.java @@ -26,7 +26,7 @@ public static SQLQueryAdapter insert(MaterializeGlobalState globalState) { MaterializeCommon.addCommonExpressionErrors(errors); errors.add("multiple assignments to same column"); errors.add("violates foreign key constraint"); - errors.add("value too long for type character varying"); + errors.add("value too long for type character"); errors.add("conflicting key value violates exclusion constraint"); errors.add("violates not-null constraint"); errors.add("current transaction is aborted"); From e039cbaa71e0dfabc69e816bf5714692f5c5e412 Mon Sep 17 00:00:00 2001 From: Elshaarawy-1 Date: Wed, 11 Feb 2026 11:36:08 +0200 Subject: [PATCH 732/856] Add DuckDBSchema.getIndexes() implementation Based on PR #1171 Addresses #1163 --- src/sqlancer/duckdb/DuckDBSchema.java | 23 ++++++++++++++++--- .../duckdb/gen/DuckDBIndexGenerator.java | 4 ++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 379576fd8..7e89da7e6 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -218,8 +218,8 @@ private static DuckDBCompositeDataType getColumnType(String typeString) { public static class DuckDBTable extends AbstractRelationalTable { - public DuckDBTable(String tableName, List columns, boolean isView) { - super(tableName, columns, Collections.emptyList(), isView); + public DuckDBTable(String tableName, List columns, List indexes, boolean isView) { + super(tableName, columns, indexes, isView); } } @@ -233,7 +233,8 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName } List databaseColumns = getTableColumns(con, tableName); boolean isView = tableName.startsWith("v"); - DuckDBTable t = new DuckDBTable(tableName, databaseColumns, isView); + List indexes = getIndexes(con, tableName, databaseName); + DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { c.setTable(t); } @@ -243,6 +244,22 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName return new DuckDBSchema(databaseTables); } + private static List getIndexes(SQLConnection con, String tableName, String databaseName) + throws SQLException { + List indexes = new ArrayList<>(); + try (Statement s = con.createStatement()) { + try (ResultSet rs = s.executeQuery(String.format( + "SELECT INDEX_NAME FROM duckdb_indexes() WHERE DATABASE_NAME = '%s' and TABLE_NAME = '%s';", + databaseName, tableName))) { + while (rs.next()) { + String indexName = rs.getString("INDEX_NAME"); + indexes.add(TableIndex.create(indexName)); + } + } + } + return indexes; + } + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 597fda19f..cc4114f63 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -23,7 +23,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append("UNIQUE "); } sb.append("INDEX "); - sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); @@ -40,7 +40,7 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } } sb.append(")"); - errors.add("already exists!"); + // errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("cannot create an index on the rowid"); } From 353edac5b4217070dd37206925c3a5823f8f62f0 Mon Sep 17 00:00:00 2001 From: Elshaarawy-1 Date: Wed, 11 Feb 2026 11:36:33 +0200 Subject: [PATCH 733/856] Fix database name filter in getIndexes() Remove DATABASE_NAME filter to fix index collision bug. --- src/sqlancer/duckdb/DuckDBSchema.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 7e89da7e6..8141220fa 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import sqlancer.IgnoreMeException; @@ -233,7 +232,7 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName } List databaseColumns = getTableColumns(con, tableName); boolean isView = tableName.startsWith("v"); - List indexes = getIndexes(con, tableName, databaseName); + List indexes = getIndexes(con, tableName); DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { c.setTable(t); @@ -244,13 +243,12 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName return new DuckDBSchema(databaseTables); } - private static List getIndexes(SQLConnection con, String tableName, String databaseName) - throws SQLException { + private static List getIndexes(SQLConnection con, String tableName) throws SQLException { List indexes = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery(String.format( - "SELECT INDEX_NAME FROM duckdb_indexes() WHERE DATABASE_NAME = '%s' and TABLE_NAME = '%s';", - databaseName, tableName))) { + "SELECT index_name FROM duckdb_indexes() WHERE database_name = current_database() AND table_name = '%s';", + tableName))) { while (rs.next()) { String indexName = rs.getString("INDEX_NAME"); indexes.add(TableIndex.create(indexName)); @@ -259,7 +257,7 @@ private static List getIndexes(SQLConnection con, String tableName, } return indexes; } - + private static List getTableNames(SQLConnection con) throws SQLException { List tableNames = new ArrayList<>(); try (Statement s = con.createStatement()) { From 5caba27dacd843f4dd70a69f4febfc35e94d97e8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 1 Mar 2026 05:02:22 +0100 Subject: [PATCH 734/856] ClickHouse: add EMPTY_LIST_OF_COLUMNS_PASSED to expected errors ClickHouse PR https://github.com/ClickHouse/ClickHouse/pull/81835 added a check that forbids creating tables without insertable columns (e.g., tables with only MATERIALIZED or ALIAS columns). Since SQLancer's random table generator can produce such definitions, ClickHouse now returns EMPTY_LIST_OF_COLUMNS_PASSED, which should be treated as an expected error rather than causing an AssertionError. The table creation retry loop in ClickHouseProvider.generateDatabase already handles expected errors by regenerating the table definition. --- src/sqlancer/clickhouse/ClickHouseErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/clickhouse/ClickHouseErrors.java b/src/sqlancer/clickhouse/ClickHouseErrors.java index 2c4d8d1b8..09fbe5ea8 100644 --- a/src/sqlancer/clickhouse/ClickHouseErrors.java +++ b/src/sqlancer/clickhouse/ClickHouseErrors.java @@ -41,6 +41,7 @@ public static List getExpectedExpressionErrors() { "Cannot convert out of range floating point value to integer type", "Unexpected inf or nan to integer conversion", "No such name in Block::erase", // https://github.com/ClickHouse/ClickHouse/issues/42769 "EMPTY_LIST_OF_COLUMNS_QUERIED", // https://github.com/ClickHouse/ClickHouse/issues/43003 + "EMPTY_LIST_OF_COLUMNS_PASSED", // https://github.com/ClickHouse/ClickHouse/pull/81835 "cannot get JOIN keys. (INVALID_JOIN_ON_EXPRESSION)", "AMBIGUOUS_IDENTIFIER", "CYCLIC_ALIASES", "Positional argument numeric constant expression is not representable as", "Positional argument must be constant with numeric type", " is out of bounds. Expected in range", From 1506cf28b48f632d29df76dd992d85f0dfa1d829 Mon Sep 17 00:00:00 2001 From: Dennis Felsing Date: Mon, 2 Mar 2026 22:40:19 +0000 Subject: [PATCH 735/856] Bump jacoco version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2037b71ce..767fedcce 100644 --- a/pom.xml +++ b/pom.xml @@ -89,7 +89,7 @@ org.jacoco jacoco-maven-plugin - 0.8.8 + 0.8.12 From d7518297ead5d46bff05552f8ecedd042afe5058 Mon Sep 17 00:00:00 2001 From: albertZhangTJ Date: Tue, 3 Mar 2026 01:37:32 +0000 Subject: [PATCH 736/856] Add expected errors for postgres 18 --- src/sqlancer/postgres/gen/PostgresCommon.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 63bc885cb..992cbc880 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -48,6 +48,7 @@ public static List getCommonTableErrors() { errors.add("is not commutative"); // exclude errors.add("operator requires run-time type coercion"); // exclude + errors.add("partitioned tables cannot be unlogged"); return errors; } @@ -59,6 +60,8 @@ public static void addCommonTableErrors(ExpectedErrors errors) { public static List getCommonExpressionErrors() { ArrayList errors = new ArrayList<>(); + errors.add("for encoding \"SQL_ASCII\" does not exist"); + errors.add("invalid byte sequence for encoding"); errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); From d7b272b93b946a66e39dc64067d5ee5b21b105f5 Mon Sep 17 00:00:00 2001 From: albertZhangTJ Date: Tue, 3 Mar 2026 02:14:40 +0000 Subject: [PATCH 737/856] Update postgres version in CI to 18 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5c53192aa..c07452653 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -504,7 +504,7 @@ jobs: - name: Set up PostgreSQL uses: harmon758/postgresql-action@v1.0.0 with: - postgresql version: '13' + postgresql version: '18' postgresql user: 'sqlancer' postgresql password: 'sqlancer' postgresql db: 'test' From a4bf0a9d65c52843d05634ca21c72f1cb1eeea72 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Sun, 22 Mar 2026 18:39:58 +0000 Subject: [PATCH 738/856] Disable SET_UNLOGGED_LOGGED for partitioned table for PG18; Fix NPE for PostgresBinaryComparisonOperation --- src/sqlancer/postgres/PostgresSchema.java | 24 ++++++++++++++++--- .../PostgresBinaryComparisonOperation.java | 2 +- .../gen/PostgresAlterTableGenerator.java | 3 +++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index c99c8648e..17089b011 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -164,6 +164,7 @@ public enum TableType { private final TableType tableType; private final List statistics; private final boolean isInsertable; + private final boolean isPartitioned; public PostgresTable(String tableName, List columns, List indexes, TableType tableType, List statistics, boolean isView, boolean isInsertable) { @@ -171,6 +172,18 @@ public PostgresTable(String tableName, List columns, List columns, List indexes, + TableType tableType, List statistics, boolean isView, boolean isInsertable, + boolean isPartitioned) { + super(tableName, columns, indexes, isView); + this.statistics = statistics; + this.isInsertable = isInsertable; + this.tableType = tableType; + this.isPartitioned = isPartitioned; } public List getStatistics() { @@ -185,6 +198,10 @@ public boolean isInsertable() { return isInsertable; } + public boolean isPartitioned() { + return isPartitioned; + } + } public static final class PostgresStatisticsObject { @@ -225,11 +242,12 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa List databaseTables = new ArrayList<>(); try (Statement s = con.createStatement()) { try (ResultSet rs = s.executeQuery( - "SELECT table_name, table_schema, table_type, is_insertable_into FROM information_schema.tables WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%' ORDER BY table_name;")) { + "SELECT t.table_name, t.table_schema, t.table_type, t.is_insertable_into, c.relkind FROM information_schema.tables t JOIN pg_class c ON c.relname = t.table_name JOIN pg_namespace n ON n.oid = c.relnamespace AND n.nspname = t.table_schema WHERE t.table_schema='public' OR t.table_schema LIKE 'pg_temp_%' ORDER BY t.table_name;")) { while (rs.next()) { String tableName = rs.getString("table_name"); String tableTypeSchema = rs.getString("table_schema"); boolean isInsertable = rs.getBoolean("is_insertable_into"); + boolean isPartitioned = "p".equals(rs.getString("relkind")); // TODO: also check insertable // TODO: insert into view? boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || @@ -240,7 +258,7 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa List indexes = getIndexes(con, tableName); List statistics = getStatistics(con); PostgresTable t = new PostgresTable(tableName, databaseColumns, indexes, tableType, statistics, - isView, isInsertable); + isView, isInsertable, isPartitioned); for (PostgresColumn c : databaseColumns) { c.setTable(t); } @@ -324,4 +342,4 @@ public String getDatabaseName() { return databaseName; } -} +} \ No newline at end of file diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 95efe5b72..daa464b77 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -126,7 +126,7 @@ public PostgresConstant getExpectedValue() { PostgresConstant leftExpectedValue = getLeft().getExpectedValue(); PostgresConstant rightExpectedValue = getRight().getExpectedValue(); if (leftExpectedValue == null || rightExpectedValue == null) { - return null; + return PostgresConstant.createNullConstant(); } return getOp().getExpectedValue(leftExpectedValue, rightExpectedValue); } diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 6e0c436ee..eb53f6df1 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -127,6 +127,9 @@ public List getActions(ExpectedErrors errors) { if (!randomTable.hasIndexes()) { action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); } + if (randomTable.isPartitioned()){ + action.remove(Action.SET_LOGGED_UNLOGGED); + } if (action.isEmpty()) { throw new IgnoreMeException(); } From 2b8796cda88cfa17856388ef57f1f8ea9989e6d6 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Sun, 22 Mar 2026 20:03:57 +0000 Subject: [PATCH 739/856] Add cannot drop inherited constraint expected error --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index eb53f6df1..d39006f3c 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -238,6 +238,8 @@ public SQLQueryAdapter generate() { errors.add("is in a primary key"); errors.add("is an identity column"); errors.add("is in index used as replica identity"); + // PG18 update: otherwise we need to encode contraint inheritance info in PostgreColumn + errors.add("cannot drop inherited constraint"); } break; case ALTER_COLUMN_SET_STATISTICS: From 21ec5c73f541a5cb85b488378caed188f184b240 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Mon, 23 Mar 2026 15:49:09 +0000 Subject: [PATCH 740/856] Format the previous commits; Update PostgresBinaryComparisonOperator null handling --- src/sqlancer/postgres/PostgresSchema.java | 2 +- .../postgres/ast/PostgresBinaryComparisonOperation.java | 3 ++- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 +- src/sqlancer/postgres/gen/PostgresCommon.java | 2 +- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 17089b011..3cc8a6e4b 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -172,7 +172,7 @@ public PostgresTable(String tableName, List columns, List getActions(ExpectedErrors errors) { if (!randomTable.hasIndexes()) { action.remove(Action.ADD_TABLE_CONSTRAINT_USING_INDEX); } - if (randomTable.isPartitioned()){ + if (randomTable.isPartitioned()) { action.remove(Action.SET_LOGGED_UNLOGGED); } if (action.isEmpty()) { diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index 992cbc880..e9f27aeb2 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -61,7 +61,7 @@ public static List getCommonExpressionErrors() { ArrayList errors = new ArrayList<>(); errors.add("for encoding \"SQL_ASCII\" does not exist"); - errors.add("invalid byte sequence for encoding"); + errors.add("invalid byte sequence for encoding"); errors.add("You might need to add explicit type casts"); errors.add("invalid regular expression"); errors.add("could not determine which collation to use"); From 9a3d65e3a10f974d9ccd4b125caaf05fba122c7f Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Mon, 23 Mar 2026 16:17:14 +0000 Subject: [PATCH 741/856] Fix checkStyle errors --- src/sqlancer/postgres/PostgresSchema.java | 3 ++- .../postgres/ast/PostgresBinaryComparisonOperation.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 3cc8a6e4b..17ef652dd 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -342,4 +342,5 @@ public String getDatabaseName() { return databaseName; } -} \ No newline at end of file +} + diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 9341d3578..3ee69ba17 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -1,7 +1,7 @@ package sqlancer.postgres.ast; -import sqlancer.Randomly; import sqlancer.IgnoreMeException; +import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.postgres.PostgresSchema.PostgresDataType; @@ -138,3 +138,4 @@ public PostgresDataType getExpressionType() { } } + From d4f459fcfb3127f609a59c71097f5c2ac4e48a27 Mon Sep 17 00:00:00 2001 From: Albert Zhang Date: Mon, 23 Mar 2026 16:19:49 +0000 Subject: [PATCH 742/856] Fix checkStyle errors --- src/sqlancer/postgres/PostgresSchema.java | 1 - src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 17ef652dd..20337fea7 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -343,4 +343,3 @@ public String getDatabaseName() { } } - diff --git a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java index 3ee69ba17..b77060dfd 100644 --- a/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java +++ b/src/sqlancer/postgres/ast/PostgresBinaryComparisonOperation.java @@ -138,4 +138,3 @@ public PostgresDataType getExpressionType() { } } - From f2b42cef4fe10e8500cc4181f241b863f1bae1ea Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 26 Mar 2026 21:52:56 +0800 Subject: [PATCH 743/856] Fix Materialize CI by removing deprecated MZ_EAT_MY_DATA flag MZ_EAT_MY_DATA=1 is a deprecated env var from old Materialize versions that triggered an initialization path running CockroachDB-style SQL (SET CLUSTER SETTING sql.stats.forecasts.enabled = false), which is no longer valid in the current Materialize architecture. Also add a pg_isready health check to ensure Materialize is accepting connections before tests run. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c3514d904..665394c8a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -392,7 +392,8 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest + until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -417,7 +418,8 @@ jobs: - name: Set up Materialize run: | docker pull materialize/materialized:latest - docker run -e MZ_EAT_MY_DATA=1 -d -p6875:6875 -p6877:6877 materialize/materialized:latest + docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest + until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 uses: actions/setup-java@v3 with: From 708af09439398b662c674ce7752f954a4a0b1295 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 08:08:49 +0800 Subject: [PATCH 744/856] Replace hardcoded sleeps with readiness loops in CI workflows Use `until` polling loops instead of fixed `sleep` durations to wait for services to become ready, reducing flakiness and unnecessary wait time across CnosDB, ClickHouse, CockroachDB, OceanBase, Presto, TiDB, YugabyteDB, and Doris jobs. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 665394c8a..b0bd5355d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -111,7 +111,7 @@ jobs: run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - sleep 5 + until curl -sf http://127.0.0.1:8902/ping 2>/dev/null; do sleep 1; done - name: Run Tests run: | CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test @@ -137,7 +137,7 @@ jobs: run: | docker pull clickhouse/clickhouse-server:24.3.1.2672 docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:24.3.1.2672 - sleep 5 + until curl -sf http://127.0.0.1:8123/ping 2>/dev/null; do sleep 1; done - name: Run Tests run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test - name: Show fatal errors @@ -166,7 +166,7 @@ jobs: run: | wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & - sleep 10 + until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests @@ -194,7 +194,7 @@ jobs: run: | wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & - sleep 10 + until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests @@ -483,7 +483,7 @@ jobs: - name: Set up OceanBase run: | docker run -p 2881:2881 --name oceanbase-ce -e MODE=mini -d oceanbase/oceanbase-ce:4.2.1-lts - sleep 120 + until mysql -h127.1 -uroot@test -P2881 --connect-timeout=3 -Doceanbase -A -e "SELECT 1" 2>/dev/null; do sleep 5; done mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | @@ -537,13 +537,13 @@ jobs: docker pull prestodb/presto:latest echo "connector.name=memory" >> memory.properties docker run -p 8080:8080 -d -v ./memory.properties:/opt/presto-server/etc/catalog/memory.properties --name presto prestodb/presto:latest - sleep 30 + until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests run: | PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoNoREC test - docker restart presto && sleep 30 + docker restart presto && until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoTLP test sqlite: name: DBMS Tests (SQLite) @@ -607,9 +607,9 @@ jobs: run: | docker pull hawkingrei/tidb-playground:nightly-2025-09-16 docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 - sleep 10 + until mysql -h 127.0.0.1 -P 4000 -u root --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done - name: Create SQLancer user - run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: | TIDB_AVAILABLE=true mvn -Dtest=TestTiDBTLP test @@ -634,9 +634,9 @@ jobs: run: | docker pull hawkingrei/tidb-playground:nightly-2025-09-16 docker run --name tidb-server -d -p 4000:4000 hawkingrei/tidb-playground:nightly-2025-09-16 - sleep 10 + until mysql -h 127.0.0.1 -P 4000 -u root --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done - name: Create SQLancer user - run: sudo mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" + run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test @@ -659,7 +659,7 @@ jobs: run: | docker pull yugabytedb/yugabyte:latest docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false - sleep 5 + until pg_isready -h localhost -p 5433; do sleep 1; done - name: Run Tests run: | YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLNoREC test @@ -696,7 +696,7 @@ jobs: cd ../be ./bin/start_be.sh --daemon - sleep 30 + until mysql -u root -h 127.0.0.1 --port 9030 --connect-timeout=3 -e "SELECT 1" 2>/dev/null; do sleep 3; done IP=$(hostname -I | awk '{print $1}') mysql -u root -h 127.0.0.1 --port 9030 -e "ALTER SYSTEM ADD BACKEND '${IP}:9050';" mysql -u root -h 127.0.0.1 --port 9030 -e "CREATE USER 'sqlancer' IDENTIFIED BY 'sqlancer'; GRANT ALL ON *.* TO sqlancer;" From 5032eae9a96fa2a37e8d6f0ff0f4f2a475a0e925 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 08:41:24 +0800 Subject: [PATCH 745/856] Update MySQL CI from 8.0.36 to 8.4 and fix compatibility issues - Update mysql service image in CI from 8.0.36 to 8.4 - Remove show_old_temporals system variable (removed in MySQL 8.4) - Update expected error for integer literals in ORDER BY: MySQL 8.4 changed the error context from 'order clause' to 'EXISTS subquery' Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 2 +- src/sqlancer/mysql/MySQLErrors.java | 2 +- src/sqlancer/mysql/gen/MySQLSetGenerator.java | 1 - src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 665394c8a..34c0e7e5d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -438,7 +438,7 @@ jobs: runs-on: ubuntu-latest services: mysql: - image: mysql:8.0.36 + image: mysql:8.4 env: MYSQL_ROOT_PASSWORD: root ports: diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index f01b30e03..bec149920 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -32,7 +32,7 @@ public static List getExpressionRegexErrors() { // "00000000000000000000-0" } - errors.add(Pattern.compile("Unknown column '.*' in 'order clause'")); + errors.add(Pattern.compile("Unknown column '.*' in 'EXISTS subquery'")); return errors; } diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 79333eb36..e350685ef 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -99,7 +99,6 @@ private enum Action { SCHEMA_DEFINITION_CACHE("schema_definition_cache", (r) -> r.getLong(256, 524288), Scope.GLOBAL), // SHOW_CREATE_TABLE_VERBOSITY("show_create_table_verbosity", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - SHOW_OLD_TEMPORALS("show_old_temporals", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), /* * sort_buffer_size is commented out as a workaround for https://bugs.mysql.com/bug.php?id=95969 */ diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index 28665c328..c613f2622 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -37,7 +37,7 @@ public class MySQLPivotedQuerySynthesisOracle public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { super(globalState); MySQLErrors.addExpressionErrors(errors); - errors.add("in 'order clause'"); // e.g., Unknown column '2067708013' in 'order clause' + errors.add("in 'EXISTS subquery'"); // e.g., Unknown column '2067708013' in 'EXISTS subquery' (MySQL 8.4+) } @Override From 227904af7a92ecfa69f5be07ddc8782fc7d3b2ef Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 09:01:12 +0800 Subject: [PATCH 746/856] Optimize CI: remove unnecessary fetch-depth and update action versions - Remove fetch-depth: 0 from all jobs (full git history is not needed for running tests) - Update actions/checkout from v3/v2 to v4 - Update actions/setup-java from v3 to v4 Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 156 +++++++++++++------------------------ 1 file changed, 52 insertions(+), 104 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 665394c8a..f0b8317e7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -24,11 +24,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -48,11 +46,9 @@ jobs: name: DBMS Tests (Citus) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -96,11 +92,9 @@ jobs: name: DBMS Tests (CnosDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -122,11 +116,9 @@ jobs: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -151,11 +143,9 @@ jobs: name: DBMS Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -179,11 +169,9 @@ jobs: name: QPG Tests (CockroachDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -213,11 +201,9 @@ jobs: - 8000:8000 - 3307:3307 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -234,9 +220,7 @@ jobs: name: DBMS Tests (DataFusion) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Rust uses: actions-rs/toolchain@v1 with: @@ -247,7 +231,7 @@ jobs: cd src/sqlancer/datafusion/server/datafusion_server cargo run & sleep 300 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -263,11 +247,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -283,11 +265,9 @@ jobs: name: DBMS Tests (H2) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -319,11 +299,9 @@ jobs: volumes: - warehouse:/opt/hive/data/warehouse steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -337,11 +315,9 @@ jobs: name: DBMS Tests (HSQLB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -366,11 +342,9 @@ jobs: options: --health-cmd="healthcheck.sh --connect --innodb_initialized" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -386,16 +360,14 @@ jobs: name: DBMS Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Materialize run: | docker pull materialize/materialized:latest docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -412,16 +384,14 @@ jobs: name: QPG Tests (Materialize) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up Materialize run: | docker pull materialize/materialized:latest docker run -d -p6875:6875 -p6877:6877 materialize/materialized:latest until pg_isready -h localhost -p 6875 -U materialize; do sleep 1; done - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -445,11 +415,9 @@ jobs: - 3306:3306 options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=10 steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -469,11 +437,9 @@ jobs: name: DBMS Tests (OceanBase) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -494,9 +460,7 @@ jobs: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up PostgreSQL uses: harmon758/postgresql-action@v1.0.0 with: @@ -505,7 +469,7 @@ jobs: postgresql password: 'sqlancer' postgresql db: 'test' - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -523,11 +487,9 @@ jobs: name: DBMS Tests (Presto) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -550,11 +512,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -573,11 +533,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -592,11 +550,9 @@ jobs: name: DBMS Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -619,11 +575,9 @@ jobs: name: QPG Tests (TiDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -644,11 +598,9 @@ jobs: name: DBMS Tests (YugabyteDB) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -671,11 +623,9 @@ jobs: name: DBMS Tests (Apache Doris) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' @@ -713,11 +663,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 + - uses: actions/checkout@v4 - name: Set up JDK 11 - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '11' From 91d7a6ae05158c1d908a1b416b08bceafb5011d0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 09:03:37 +0800 Subject: [PATCH 747/856] Restore 'order clause' error alongside 'EXISTS subquery' for MySQL 8.4 MySQL 8.4 uses 'order clause' when the ORDER BY only contains an integer literal, and 'EXISTS subquery' when ORDER BY also contains an EXISTS expression. Both patterns need to be handled. Co-Authored-By: Claude Sonnet 4.6 --- src/sqlancer/mysql/MySQLErrors.java | 1 + src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index bec149920..989c8fed6 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -32,6 +32,7 @@ public static List getExpressionRegexErrors() { // "00000000000000000000-0" } + errors.add(Pattern.compile("Unknown column '.*' in 'order clause'")); errors.add(Pattern.compile("Unknown column '.*' in 'EXISTS subquery'")); return errors; diff --git a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java index c613f2622..c1fe893b6 100644 --- a/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLPivotedQuerySynthesisOracle.java @@ -37,6 +37,7 @@ public class MySQLPivotedQuerySynthesisOracle public MySQLPivotedQuerySynthesisOracle(MySQLGlobalState globalState) throws SQLException { super(globalState); MySQLErrors.addExpressionErrors(errors); + errors.add("in 'order clause'"); // e.g., Unknown column '2067708013' in 'order clause' errors.add("in 'EXISTS subquery'"); // e.g., Unknown column '2067708013' in 'EXISTS subquery' (MySQL 8.4+) } From 8c143a3d4951bf85d891a0fbbefcfa0f08ff38d9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 27 Mar 2026 09:09:47 +0800 Subject: [PATCH 748/856] Fix CnosDB readiness check to use TCP port probe The /ping endpoint does not exist on CnosDB, causing curl -f to loop forever. Switch to nc -z which just checks that the port is open. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0bd5355d..6bd47d3e1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -111,7 +111,7 @@ jobs: run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - until curl -sf http://127.0.0.1:8902/ping 2>/dev/null; do sleep 1; done + until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done - name: Run Tests run: | CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test From 6837dd1d05e7da8ef49a577a6b58498c03da9325 Mon Sep 17 00:00:00 2001 From: Carmen Kwan Date: Thu, 2 Apr 2026 01:12:40 +0200 Subject: [PATCH 749/856] Fix CONTRIBUTING.md Stop referencing files that don't exist --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5db3bdc69..ea5baea1c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,7 @@ If you do not find an option to import Maven projects, you might need to install ## Implementing Support for a New DBMS -The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see `DuckDBNoRECOracle` or `DuckDBQueryPartitioningWhereTester` for TLP). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). +The DuckDB implementation provides a good template for a new implementation. The `DuckDBProvider` class is the central class that manages the creation of the databases and executes the selected test oracles. Try to copy its structure for the new DBMS that you want to implement, and start by generate databases (without implementing a test oracle). As part of this, you will also need to implement the equivalent of `DuckDBSchema`, which represents the database schema of the generated database. After you can successfully generate databases, the next step is to generate one of the test oracles. For example, you might want to implement NoREC (see enum value `NOREC` in `DuckDBOracleFactory`). As part of this, you must also implement a random expression generator (see `DuckDBExpressionGenerator`) and a visitor to derive the textual representation of an expression (see `DuckDBToStringVisitor`). Please consider the following suggestions when creating a PR to contribute a new DBMS: * Ensure that `mvn verify -DskipTests=true` does not result in style violations. From 8787427efc005358f83913ace3fc1beaf2350c2e Mon Sep 17 00:00:00 2001 From: Aman Bihari <161295600+codebreaker32@users.noreply.github.com> Date: Fri, 3 Apr 2026 06:16:57 +0000 Subject: [PATCH 750/856] Remove outdated comment and wrong errors --- src/sqlancer/spark/SparkErrors.java | 2 -- src/sqlancer/spark/SparkProvider.java | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java index 97c8056a3..83ba84bb3 100644 --- a/src/sqlancer/spark/SparkErrors.java +++ b/src/sqlancer/spark/SparkErrors.java @@ -35,8 +35,6 @@ public static List getExpressionErrors() { errors.add("expression is neither present in the group by"); errors.add("is not a valid grouping expression"); errors.add("is not contained in either an aggregate function or the GROUP BY clause"); - errors.add("PARSE_SYNTAX_ERROR"); - errors.add("Syntax error"); return errors; } diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java index f53ca10a8..89f24a738 100644 --- a/src/sqlancer/spark/SparkProvider.java +++ b/src/sqlancer/spark/SparkProvider.java @@ -28,7 +28,7 @@ public SparkProvider() { } public enum Action implements AbstractAction { - INSERT(SparkInsertGenerator::getQuery); // You will need to create this class + INSERT(SparkInsertGenerator::getQuery); private final SQLQueryProvider sqlQueryProvider; From 787fcf3225a313059c92dadf3cab25ea6cbc120e Mon Sep 17 00:00:00 2001 From: Aman Date: Sat, 4 Apr 2026 15:05:35 +0530 Subject: [PATCH 751/856] Format spark/* --- src/sqlancer/Main.java | 2 +- src/sqlancer/spark/SparkErrors.java | 2 +- src/sqlancer/spark/SparkGlobalState.java | 2 +- src/sqlancer/spark/SparkOptions.java | 2 +- src/sqlancer/spark/SparkProvider.java | 5 ++-- src/sqlancer/spark/SparkSchema.java | 24 ++++++++++++------- src/sqlancer/spark/SparkToStringVisitor.java | 5 ++-- .../spark/ast/SparkBetweenOperation.java | 2 +- .../spark/ast/SparkBinaryOperation.java | 2 +- .../spark/ast/SparkCaseOperation.java | 2 +- .../spark/ast/SparkCastOperation.java | 2 +- .../spark/ast/SparkColumnReference.java | 2 +- src/sqlancer/spark/ast/SparkConstant.java | 2 +- src/sqlancer/spark/ast/SparkExpression.java | 2 +- src/sqlancer/spark/ast/SparkInOperation.java | 2 +- src/sqlancer/spark/ast/SparkJoin.java | 2 +- src/sqlancer/spark/ast/SparkOrderingTerm.java | 2 +- src/sqlancer/spark/ast/SparkSelect.java | 2 +- .../spark/ast/SparkTableReference.java | 2 +- .../spark/ast/SparkUnaryPostfixOperation.java | 2 +- .../spark/ast/SparkUnaryPrefixOperation.java | 2 +- .../spark/gen/SparkExpressionGenerator.java | 2 +- .../spark/gen/SparkInsertGenerator.java | 2 +- .../spark/gen/SparkTableGenerator.java | 8 ++++--- 24 files changed, 47 insertions(+), 35 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index f778bd7da..faf35e3c9 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -44,11 +44,11 @@ import sqlancer.postgres.PostgresProvider; import sqlancer.presto.PrestoProvider; import sqlancer.questdb.QuestDBProvider; +import sqlancer.spark.SparkProvider; import sqlancer.sqlite3.SQLite3Provider; import sqlancer.tidb.TiDBProvider; import sqlancer.yugabyte.ycql.YCQLProvider; import sqlancer.yugabyte.ysql.YSQLProvider; -import sqlancer.spark.SparkProvider; public final class Main { diff --git a/src/sqlancer/spark/SparkErrors.java b/src/sqlancer/spark/SparkErrors.java index 83ba84bb3..a3a96061f 100644 --- a/src/sqlancer/spark/SparkErrors.java +++ b/src/sqlancer/spark/SparkErrors.java @@ -62,4 +62,4 @@ public static List getInsertErrors() { public static void addInsertErrors(ExpectedErrors errors) { errors.addAll(getInsertErrors()); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkGlobalState.java b/src/sqlancer/spark/SparkGlobalState.java index e79826332..d78c737e5 100644 --- a/src/sqlancer/spark/SparkGlobalState.java +++ b/src/sqlancer/spark/SparkGlobalState.java @@ -8,4 +8,4 @@ public class SparkGlobalState extends SQLGlobalState protected SparkSchema readSchema() throws Exception { return SparkSchema.fromConnection(getConnection(), getDatabaseName()); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkOptions.java b/src/sqlancer/spark/SparkOptions.java index c9422a910..7b347ceef 100644 --- a/src/sqlancer/spark/SparkOptions.java +++ b/src/sqlancer/spark/SparkOptions.java @@ -40,4 +40,4 @@ public TestOracle create(SparkGlobalState globalState) throws public List getTestOracleFactory() { return oracle; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkProvider.java b/src/sqlancer/spark/SparkProvider.java index 89f24a738..817a92471 100644 --- a/src/sqlancer/spark/SparkProvider.java +++ b/src/sqlancer/spark/SparkProvider.java @@ -109,7 +109,8 @@ public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLExce con = DriverManager.getConnection(String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName), username, password); try (Statement s = con.createStatement()) { - // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the generator loves to do. + // This allows casting things like BOOLEAN to DATE/TIMESTAMP, which the + // generator loves to do. s.execute("SET spark.sql.ansi.enabled=false"); } return new SQLConnection(con); @@ -119,4 +120,4 @@ public SQLConnection createDatabase(SparkGlobalState globalState) throws SQLExce public String getDBMSName() { return "spark"; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java index 849652b19..8d988e31e 100644 --- a/src/sqlancer/spark/SparkSchema.java +++ b/src/sqlancer/spark/SparkSchema.java @@ -85,8 +85,9 @@ private static List getTableColumns(SQLConnection con, String table String columnName = rs.getString("col_name"); String dataType = rs.getString("data_type"); // Filter out Spark partition info or comments usually at bottom of describe - if (columnName.startsWith("#") || columnName.isEmpty()) + if (columnName.startsWith("#") || columnName.isEmpty()) { continue; + } columns.add(new SparkColumn(columnName, null, getColumnType(dataType))); } @@ -97,18 +98,25 @@ private static List getTableColumns(SQLConnection con, String table private static SparkDataType getColumnType(String typeString) { String upper = typeString.toUpperCase(); - if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) + if (upper.startsWith("STRING") || upper.startsWith("VARCHAR") || upper.startsWith("CHAR")) { return SparkDataType.STRING; - if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) + } + if (upper.startsWith("INT") || upper.startsWith("BIGINT") || upper.startsWith("SMALLINT")) { return SparkDataType.INTEGER; - if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) + } + if (upper.startsWith("DOUBLE") || upper.startsWith("FLOAT") || upper.startsWith("DECIMAL")) { return SparkDataType.DOUBLE; - if (upper.startsWith("BOOLEAN")) + } + if (upper.startsWith("BOOLEAN")) { return SparkDataType.BOOLEAN; - if (upper.startsWith("TIMESTAMP")) + } + if (upper.startsWith("TIMESTAMP")) { return SparkDataType.TIMESTAMP; - if (upper.startsWith("DATE")) + } + if (upper.startsWith("DATE")) { return SparkDataType.DATE; + } return SparkDataType.STRING; // Fallback } -} \ No newline at end of file + +} diff --git a/src/sqlancer/spark/SparkToStringVisitor.java b/src/sqlancer/spark/SparkToStringVisitor.java index 91f47e32c..0777c86a6 100644 --- a/src/sqlancer/spark/SparkToStringVisitor.java +++ b/src/sqlancer/spark/SparkToStringVisitor.java @@ -63,7 +63,8 @@ private void visit(SparkSelect select) { sb.append(" LIMIT "); visit(select.getLimitClause()); } - // Spark supports OFFSET, though strictly usually with LIMIT or in newer versions + // Spark supports OFFSET, though strictly usually with LIMIT or in newer + // versions if (select.getOffsetClause() != null) { sb.append(" OFFSET "); visit(select.getOffsetClause()); @@ -117,4 +118,4 @@ public static String asString(SparkExpression expr) { visitor.visit(expr); return visitor.get(); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkBetweenOperation.java b/src/sqlancer/spark/ast/SparkBetweenOperation.java index f229c1c7c..59297ba8f 100644 --- a/src/sqlancer/spark/ast/SparkBetweenOperation.java +++ b/src/sqlancer/spark/ast/SparkBetweenOperation.java @@ -7,4 +7,4 @@ public class SparkBetweenOperation extends NewBetweenOperatorNode public SparkBinaryOperation(SparkExpression left, SparkExpression right, Operator op) { super(left, right, op); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkCaseOperation.java b/src/sqlancer/spark/ast/SparkCaseOperation.java index fb1ee0cd8..995fd7f52 100644 --- a/src/sqlancer/spark/ast/SparkCaseOperation.java +++ b/src/sqlancer/spark/ast/SparkCaseOperation.java @@ -10,4 +10,4 @@ public SparkCaseOperation(SparkExpression switchCondition, List List expressions, SparkExpression elseExpr) { super(switchCondition, conditions, expressions, elseExpr); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkCastOperation.java b/src/sqlancer/spark/ast/SparkCastOperation.java index 3bc5eb30d..547551285 100644 --- a/src/sqlancer/spark/ast/SparkCastOperation.java +++ b/src/sqlancer/spark/ast/SparkCastOperation.java @@ -22,4 +22,4 @@ public SparkExpression getExpression() { public SparkDataType getType() { return type; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkColumnReference.java b/src/sqlancer/spark/ast/SparkColumnReference.java index 75e92d267..ccd1b7855 100644 --- a/src/sqlancer/spark/ast/SparkColumnReference.java +++ b/src/sqlancer/spark/ast/SparkColumnReference.java @@ -8,4 +8,4 @@ public class SparkColumnReference extends ColumnReferenceNode { -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkInOperation.java b/src/sqlancer/spark/ast/SparkInOperation.java index 37a80e3ff..430d9b5c2 100644 --- a/src/sqlancer/spark/ast/SparkInOperation.java +++ b/src/sqlancer/spark/ast/SparkInOperation.java @@ -9,4 +9,4 @@ public class SparkInOperation extends NewInOperatorNode impleme public SparkInOperation(SparkExpression left, List right, boolean isNegated) { super(left, right, isNegated); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkJoin.java b/src/sqlancer/spark/ast/SparkJoin.java index 44da7fba4..a59eaff48 100644 --- a/src/sqlancer/spark/ast/SparkJoin.java +++ b/src/sqlancer/spark/ast/SparkJoin.java @@ -43,4 +43,4 @@ public SparkExpression getOnClause() { public void setOnClause(SparkExpression onClause) { this.onClause = onClause; } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkOrderingTerm.java b/src/sqlancer/spark/ast/SparkOrderingTerm.java index 824801c00..870c8239b 100644 --- a/src/sqlancer/spark/ast/SparkOrderingTerm.java +++ b/src/sqlancer/spark/ast/SparkOrderingTerm.java @@ -7,4 +7,4 @@ public class SparkOrderingTerm extends NewOrderingTerm implemen public SparkOrderingTerm(SparkExpression expr, Ordering ordering) { super(expr, ordering); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkSelect.java b/src/sqlancer/spark/ast/SparkSelect.java index 0986ce0a6..8b59f5513 100644 --- a/src/sqlancer/spark/ast/SparkSelect.java +++ b/src/sqlancer/spark/ast/SparkSelect.java @@ -39,4 +39,4 @@ public String asString() { return SparkToStringVisitor.asString(this); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkTableReference.java b/src/sqlancer/spark/ast/SparkTableReference.java index 92a59ad3d..5bcbb5d03 100644 --- a/src/sqlancer/spark/ast/SparkTableReference.java +++ b/src/sqlancer/spark/ast/SparkTableReference.java @@ -10,4 +10,4 @@ public SparkTableReference(SparkSchema.SparkTable table) { super(table); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java index f1082a655..3dd9d28e2 100644 --- a/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java +++ b/src/sqlancer/spark/ast/SparkUnaryPostfixOperation.java @@ -10,4 +10,4 @@ public SparkUnaryPostfixOperation(SparkExpression expr, Operator op) { super(expr, op); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java index d1bd94ab4..5c1a8e4c6 100644 --- a/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java +++ b/src/sqlancer/spark/ast/SparkUnaryPrefixOperation.java @@ -9,4 +9,4 @@ public SparkUnaryPrefixOperation(SparkExpression expr, Operator op) { super(expr, op); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/gen/SparkExpressionGenerator.java b/src/sqlancer/spark/gen/SparkExpressionGenerator.java index faf8a07f0..3708f314a 100644 --- a/src/sqlancer/spark/gen/SparkExpressionGenerator.java +++ b/src/sqlancer/spark/gen/SparkExpressionGenerator.java @@ -333,4 +333,4 @@ public int getNrArgs() { } } } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java index 29232fdb2..b1755a848 100644 --- a/src/sqlancer/spark/gen/SparkInsertGenerator.java +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -44,4 +44,4 @@ private SQLQueryAdapter generate() { SparkErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, false, false); } -} \ No newline at end of file +} diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java index 68cafdafb..937e52248 100644 --- a/src/sqlancer/spark/gen/SparkTableGenerator.java +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -19,8 +19,10 @@ public class SparkTableGenerator { private enum ColumnConstraints { NOT_NULL, DEFAULT - // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources (Parquet/ORC) - // without specific catalogs (like Delta/Iceberg), so we limit to constraints Spark SQL widely accepts. + // PRIMARY KEY and UNIQUE are often not supported in standard Spark file sources + // (Parquet/ORC) + // without specific catalogs (like Delta/Iceberg), so we limit to constraints + // Spark SQL widely accepts. } private final SparkGlobalState globalState; @@ -97,4 +99,4 @@ private void appendColumnConstraint() { throw new AssertionError(constraint); } } -} \ No newline at end of file +} From 22d5c1335d705fb9992ce347728ce90b7423a1cb Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:04:14 +0800 Subject: [PATCH 752/856] Fix DataFusion CI: replace blind sleep with build step and readiness poll The DataFusion CI job was flaky because it backgrounded `cargo run` with a fixed 300s sleep that raced against compilation time. Split into explicit build, start, and readiness-check steps so tests only run once the server is actually listening on port 50051. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 34c0e7e5d..d22c46ff9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -242,10 +242,14 @@ jobs: with: toolchain: stable override: true - - name: Run DataFusion Server + - name: Build DataFusion Server run: | cd src/sqlancer/datafusion/server/datafusion_server - cargo run & sleep 300 + cargo build + - name: Start DataFusion Server + run: | + cd src/sqlancer/datafusion/server/datafusion_server + cargo run & - name: Set up JDK 11 uses: actions/setup-java@v3 with: @@ -254,6 +258,18 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true + - name: Wait for DataFusion Server + run: | + for i in $(seq 1 30); do + if nc -z 127.0.0.1 50051 2>/dev/null; then + echo "DataFusion server is ready" + exit 0 + fi + echo "Waiting for DataFusion server... ($i/30)" + sleep 10 + done + echo "DataFusion server failed to start within 300s" + exit 1 - name: Run Tests run: | DATAFUSION_AVAILABLE=true mvn test -Pdatafusion-tests From 9d26d70264ea85379830ec8a6907321ccbea6be1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:12:16 +0800 Subject: [PATCH 753/856] Pin chrono <0.4.40 to fix arrow-arith 52.2.0 build conflict chrono 0.4.40+ added `Datelike::quarter()` which conflicts with `ChronoDateExt::quarter()` in arrow-arith 52.2.0, causing ambiguous method resolution. Pin chrono below 0.4.40 until arrow dependencies are upgraded. Co-Authored-By: Claude Sonnet 4.6 --- src/sqlancer/datafusion/server/datafusion_server/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml index cd8b85e1d..332a88e30 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml +++ b/src/sqlancer/datafusion/server/datafusion_server/Cargo.toml @@ -16,7 +16,7 @@ arrow-schema = { version = "52.1.0", default-features = false } arrow-string = { version = "52.1.0", default-features = false } async-trait = "0.1.73" bytes = "1.4" -chrono = { version = "0.4.34", default-features = false } +chrono = { version = ">=0.4.34, <0.4.40", default-features = false } dashmap = "5.5.0" # This version is for SQLancer CI run datafusion = { version = "40.0.0" } From c548689d99bc3d9f5290d2ca28ec5965964c1405 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:32:02 +0800 Subject: [PATCH 754/856] Fix DataFusion server memory leak: clear state on database reset The server's DashMaps (contexts, statements, results) were never cleaned up between SQLancer rounds, causing unbounded memory growth that eventually stalled query throughput to 0/s. Co-Authored-By: Claude Sonnet 4.6 --- src/sqlancer/datafusion/server/datafusion_server/src/main.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs index 13ec73e96..057c34883 100644 --- a/src/sqlancer/datafusion/server/datafusion_server/src/main.rs +++ b/src/sqlancer/datafusion/server/datafusion_server/src/main.rs @@ -215,6 +215,11 @@ impl FlightSqlService for FlightSqlServiceImpl { let mut ctx_guard = self.ctx.lock().await; // Use `lock()` for async Mutex *ctx_guard = new_ctx; + + // Clear leaked state from previous round + self.statements.clear(); + self.results.clear(); + self.contexts.clear(); } // no authentication actually takes place here // see Ballista implementation for example of basic auth From 5b4f606ad4a9c83ff638e4329b8b53144651f738 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 00:43:34 +0800 Subject: [PATCH 755/856] Fix HSQLDB test: resolve SLF4J/Log4j version conflicts The HSQLDB test failed with NoClassDefFoundError on FrameworkLogger due to incompatible logging dependencies from Hive: - log4j-slf4j-impl (SLF4J 1.x bridge) conflicting with slf4j-api 2.0.6 - log4j-api 2.10.0 and log4j-core 2.18.0 version mismatch Fix by excluding log4j-slf4j-impl from all Hive dependencies, aligning Log4j2 at 2.24.3, and adding log4j-slf4j2-impl (SLF4J 2.x compatible). Also upgrade HSQLDB from 2.7.1 to 2.7.4. Co-Authored-By: Claude Sonnet 4.6 --- pom.xml | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 0fe88c17d..7c9a1106b 100644 --- a/pom.xml +++ b/pom.xml @@ -360,7 +360,7 @@ org.hsqldb hsqldb - 2.7.1 + 2.7.4 runtime @@ -382,16 +382,49 @@ org.apache.hive hive-jdbc 3.1.2 + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.hive hive-serde 4.0.1 + + + org.apache.logging.log4j + log4j-slf4j-impl + + org.apache.hive hive-cli 4.0.1 + + + org.apache.logging.log4j + log4j-slf4j-impl + + + + + org.apache.logging.log4j + log4j-api + 2.24.3 + + + org.apache.logging.log4j + log4j-core + 2.24.3 + + + org.apache.logging.log4j + log4j-slf4j2-impl + 2.24.3 org.apache.hadoop From f928e7c98038097fcbcbde1cff85cf5d3549240a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 14:41:36 +0800 Subject: [PATCH 756/856] Fix CI job names: typos, copy-paste errors, and creation-only labels - CnosDB: fix step name "Set up ClickHouse" -> "Set up CnosDB" - CnosDB: add "creation only" (both NoREC and TLP use --num-queries 0) - HSQLDB: fix typo "HSQLB" -> "HSQLDB" - MySQL: note "CERT creation only" (CERT uses --num-queries 0) - TiDB: note "TLP creation only" (TLP uses --num-queries 0) Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7f2e493f9..ed6769c34 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -89,7 +89,7 @@ jobs: run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test cnosdb: - name: DBMS Tests (CnosDB) + name: DBMS Tests (CnosDB, creation only) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -101,7 +101,7 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Set up ClickHouse + - name: Set up CnosDB run: | docker pull cnosdb/cnosdb:community-latest docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest @@ -366,7 +366,7 @@ jobs: run: SPARK_AVAILABLE=true mvn -Dtest=TestSparkTLP test hsqldb: - name: DBMS Tests (HSQLB) + name: DBMS Tests (HSQLDB) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -458,7 +458,7 @@ jobs: MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQueryPlan mysql: - name: DBMS Tests (MySQL) + name: DBMS Tests (MySQL, CERT creation only) runs-on: ubuntu-latest services: mysql: @@ -601,7 +601,7 @@ jobs: mvn -Dtest=TestSQLiteQPG test tidb: - name: DBMS Tests (TiDB) + name: DBMS Tests (TiDB, TLP creation only) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 From 6a352e88c489f145c617c58f64eb1f4180b9a717 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 5 Apr 2026 23:45:44 +0800 Subject: [PATCH 757/856] Remove unnecessary "does not exist" expected error from PostgresReindexGenerator Following the REINDEX syntax fix in PR #1283 (89a48dd2), the "does not exist" expected error is no longer needed. It was only masking the previous bug where multiple index names were concatenated without delimiters (e.g., REINDEX INDEX i0i1i2), causing PostgreSQL to report that the concatenated name does not exist. With the fix now selecting a single valid index, this error cannot legitimately occur since getIndexes() only returns indexes that exist in the schema. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresReindexGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java index dc0d2cf34..d22ffe53e 100644 --- a/src/sqlancer/postgres/gen/PostgresReindexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresReindexGenerator.java @@ -58,7 +58,6 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { throw new AssertionError(scope); } errors.add("already contains data"); // FIXME bug report - errors.add("does not exist"); // internal index errors.add("REINDEX is not yet implemented for partitioned indexes"); return new SQLQueryAdapter(sb.toString(), errors); } From ce741c9eaa9f28130af5479a1e95f979a097d391 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 6 Apr 2026 09:53:54 +0800 Subject: [PATCH 758/856] Add EXPLAIN with randomized options for Postgres Based on PR #1265, which adds GENERIC_PLAN support for EXPLAIN. This reimplements the feature with the following bugs fixed: - EXPLAIN options were each wrapped in separate parentheses (e.g. EXPLAIN (ANALYZE) (FORMAT JSON) ...) instead of a single comma-separated list (EXPLAIN (ANALYZE, FORMAT JSON) ...) - BUFFERS and TIMING were generated without ANALYZE, which is required - GENERIC_PLAN and ANALYZE were not treated as mutually exclusive Closes #1044 point 3 Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/PostgresProvider.java | 4 ++ .../gen/PostgresExplainGenerator.java | 68 ++++++++++++++++++- 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index acd20a184..ec7978216 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -127,6 +127,7 @@ public enum Action implements AbstractAction { LISTEN((g) -> PostgresNotifyGenerator.createListen()), // UNLISTEN((g) -> PostgresNotifyGenerator.createUnlisten()), // CREATE_SEQUENCE(PostgresSequenceGenerator::createSequence), // + EXPLAIN(PostgresExplainGenerator::create), // CREATE_VIEW(PostgresViewGenerator::create), // CREATE_TABLESPACE(PostgresTableSpaceGenerator::generate); @@ -201,6 +202,9 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { case INSERT: nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); break; + case EXPLAIN: + nrPerformed = r.getInteger(0, 1); + break; default: throw new AssertionError(a); } diff --git a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java index d3039394b..e4359e5aa 100644 --- a/src/sqlancer/postgres/gen/PostgresExplainGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresExplainGenerator.java @@ -1,16 +1,82 @@ package sqlancer.postgres.gen; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresSchema; +import sqlancer.postgres.PostgresSchema.PostgresDataType; +import sqlancer.postgres.PostgresSchema.PostgresTables; +import sqlancer.postgres.ast.PostgresSelect; + public final class PostgresExplainGenerator { private PostgresExplainGenerator() { } - public static String explain(String selectStr) throws Exception { + public static String explain(String selectStr) { StringBuilder sb = new StringBuilder(); sb.append("EXPLAIN (FORMAT JSON) "); sb.append(selectStr); return sb.toString(); } + public static String explainGeneral(String selectStr) { + StringBuilder sb = new StringBuilder(); + sb.append("EXPLAIN "); + + List options = new ArrayList<>(); + boolean analyze = Randomly.getBoolean(); + boolean genericPlan = !analyze && Randomly.getBoolean(); + if (analyze) { + options.add("ANALYZE"); + } + if (genericPlan) { + options.add("GENERIC_PLAN"); + } + if (Randomly.getBoolean()) { + options.add("FORMAT " + Randomly.fromOptions("TEXT", "XML", "JSON", "YAML")); + } + if (Randomly.getBoolean()) { + options.add("VERBOSE"); + } + if (Randomly.getBoolean()) { + options.add("COSTS"); + } + if (analyze && Randomly.getBoolean()) { + options.add("BUFFERS"); + } + if (analyze && Randomly.getBoolean()) { + options.add("TIMING"); + } + if (Randomly.getBoolean()) { + options.add("SUMMARY"); + } + if (!options.isEmpty()) { + sb.append("("); + sb.append(String.join(", ", options)); + sb.append(") "); + } + + sb.append(selectStr); + return sb.toString(); + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) throws Exception { + PostgresSchema.PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + PostgresExpressionGenerator gen = new PostgresExpressionGenerator(globalState); + gen.setTablesAndColumns(new PostgresTables(Arrays.asList(table))); + PostgresSelect select = gen.generateSelect(); + select.setFromList(gen.getTableRefs()); + select.setFetchColumns(gen.generateFetchColumns(false)); + if (Randomly.getBoolean()) { + select.setWhereClause(gen.generateExpression(PostgresDataType.BOOLEAN)); + } + return new SQLQueryAdapter(explainGeneral(select.asString())); + } + } From c67c7380ee39fa685895eea7839929aad41d44e4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Apr 2026 09:32:27 +0800 Subject: [PATCH 759/856] PostgreSQL: Add expected error for PRIMARY KEY on column with DROP NOT NULL When a multi-action ALTER TABLE combines ADD CONSTRAINT ... PRIMARY KEY with ALTER ... DROP NOT NULL, PostgreSQL returns "primary key column is not marked NOT NULL". Add this to expected errors in both ADD_TABLE_CONSTRAINT and ADD_TABLE_CONSTRAINT_USING_INDEX cases. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 5598c3f9f..69b509f60 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -304,6 +304,7 @@ public SQLQueryAdapter generate() { errors.add("multiple primary keys for table"); errors.add("could not create unique index"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("cannot cast type"); errors.add("unsupported PRIMARY KEY constraint with partition key definition"); errors.add("unsupported UNIQUE constraint with partition key definition"); @@ -342,6 +343,7 @@ public SQLQueryAdapter generate() { errors.add("appears twice in unique constraint"); errors.add("appears twice in primary key constraint"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("insufficient columns in PRIMARY KEY constraint definition"); errors.add("which is part of the partition key"); break; From 3db5ffa9c2153bcf5c36ea6278177e20063538b7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 7 Apr 2026 09:34:42 +0800 Subject: [PATCH 760/856] PostgreSQL: Add expected error for PRIMARY KEY on column with DROP NOT NULL When a multi-action ALTER TABLE combines ADD CONSTRAINT ... PRIMARY KEY with ALTER ... DROP NOT NULL, PostgreSQL returns "primary key column is not marked NOT NULL". Add this to expected errors in both ADD_TABLE_CONSTRAINT and ADD_TABLE_CONSTRAINT_USING_INDEX cases. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java index 5598c3f9f..69b509f60 100644 --- a/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresAlterTableGenerator.java @@ -304,6 +304,7 @@ public SQLQueryAdapter generate() { errors.add("multiple primary keys for table"); errors.add("could not create unique index"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("cannot cast type"); errors.add("unsupported PRIMARY KEY constraint with partition key definition"); errors.add("unsupported UNIQUE constraint with partition key definition"); @@ -342,6 +343,7 @@ public SQLQueryAdapter generate() { errors.add("appears twice in unique constraint"); errors.add("appears twice in primary key constraint"); errors.add("contains null values"); + errors.add("is not marked NOT NULL"); errors.add("insufficient columns in PRIMARY KEY constraint definition"); errors.add("which is part of the partition key"); break; From d8d90bbfefccd567270bda1de81b3dd0ff00f5ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 8 Apr 2026 00:35:01 +0800 Subject: [PATCH 761/856] Remove commented-out code in the DuckDB index generator --- src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index cc4114f63..6c50b204d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -40,7 +40,6 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } } sb.append(")"); - // errors.add("already exists!"); if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("cannot create an index on the rowid"); } From 6d8685090321a11648e2dd2f8538aadb02f937e4 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 8 Apr 2026 09:55:17 +0800 Subject: [PATCH 762/856] Materialize: Update expected query plan for QPG test The latest Materialize version changed the EXPLAIN OPTIMIZED PLAN output format: the With block now appears before Return, and column name annotations shifted positions. Update the expected string to match. Co-Authored-By: Claude Opus 4.6 --- test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java index f282af42e..4d26bc08e 100644 --- a/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java +++ b/test/sqlancer/qpg/materialize/TestMaterializeQueryPlan.java @@ -42,7 +42,7 @@ void testMaterializeQueryPlan() throws Exception { String queryPlan = provider.getQueryPlan("SELECT * FROM t1 RIGHT JOIN t2 ON a<>0;", state); assertEquals( - "Return // { arity: 3 };Union // { arity: 3 };Get l0 // { arity: 3 };Project (#2{c}, #3, #0) // { arity: 3 };Union // { arity: 1 };Negate // { arity: 1 };Project (#2) // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };With;ReadStorage queryplan.public.t1 // { arity: 2 };ReadStorage queryplan.public.t2 // { arity: 1 };;Source queryplan.public.t1;Source queryplan.public.t2;;Target cluster: quickstart;", + "With;ReadStorage queryplan.public.t1 // { arity: 2 };ReadStorage queryplan.public.t2 // { arity: 1 };Return // { arity: 3 };Union // { arity: 3 };Get l0 // { arity: 3 };Project (#2, #3, #0{c}) // { arity: 3 };Union // { arity: 1 };Negate // { arity: 1 };Project (#2{c}) // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };ReadStorage queryplan.public.t2 // { arity: 1 };;Source queryplan.public.t1;Source queryplan.public.t2;;Target cluster: quickstart;", queryPlan); } From 9a9b5f345aa26571a815928ff96a24e41ffeee85 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Apr 2026 00:09:46 +0800 Subject: [PATCH 763/856] Use AbstractSchema.matchesViewName() for view detection everywhere Replace inlined tableName.startsWith("v") with the existing matchesViewName() helper across all 15 schema classes, ensuring a single source of truth for view name detection. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/clickhouse/ClickHouseSchema.java | 2 +- src/sqlancer/cockroachdb/CockroachDBSchema.java | 2 +- src/sqlancer/databend/DatabendSchema.java | 2 +- src/sqlancer/datafusion/DataFusionSchema.java | 2 +- src/sqlancer/doris/DorisSchema.java | 2 +- src/sqlancer/duckdb/DuckDBSchema.java | 2 +- src/sqlancer/hive/HiveSchema.java | 2 +- src/sqlancer/hsqldb/HSQLDBSchema.java | 2 +- src/sqlancer/postgres/PostgresSchema.java | 2 +- src/sqlancer/presto/PrestoSchema.java | 2 +- src/sqlancer/questdb/QuestDBSchema.java | 2 +- src/sqlancer/spark/SparkSchema.java | 2 +- src/sqlancer/tidb/TiDBSchema.java | 2 +- src/sqlancer/yugabyte/ycql/YCQLSchema.java | 2 +- src/sqlancer/yugabyte/ysql/YSQLSchema.java | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/sqlancer/clickhouse/ClickHouseSchema.java b/src/sqlancer/clickhouse/ClickHouseSchema.java index 97f4534a1..8f8f906ec 100644 --- a/src/sqlancer/clickhouse/ClickHouseSchema.java +++ b/src/sqlancer/clickhouse/ClickHouseSchema.java @@ -216,7 +216,7 @@ public static ClickHouseSchema fromConnection(SQLConnection con, String database for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); List indexes = Collections.emptyList(); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); ClickHouseTable t = new ClickHouseTable(tableName, databaseColumns, indexes, isView); for (ClickHouseColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 708afef92..c0eeedd2b 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -292,7 +292,7 @@ public static CockroachDBSchema fromConnection(SQLConnection con, String databas for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); List indexes = getIndexes(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); CockroachDBTable t = new CockroachDBTable(tableName, databaseColumns, indexes, isView); for (CockroachDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index af9a3e0ce..f6bf30757 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -321,7 +321,7 @@ public static DatabendSchema fromConnection(SQLConnection con, String databaseNa List tableNames = getTableNames(con, databaseName); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName, databaseName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); DatabendTable t = new DatabendTable(tableName, databaseColumns, isView); for (DatabendColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/datafusion/DataFusionSchema.java b/src/sqlancer/datafusion/DataFusionSchema.java index b9f00a02e..d02e80c30 100644 --- a/src/sqlancer/datafusion/DataFusionSchema.java +++ b/src/sqlancer/datafusion/DataFusionSchema.java @@ -37,7 +37,7 @@ public static DataFusionSchema fromConnection(SQLConnection con, String database for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); DataFusionTable t = new DataFusionTable(tableName, databaseColumns, isView); for (DataFusionColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/doris/DorisSchema.java b/src/sqlancer/doris/DorisSchema.java index 76697fad7..70a61ee62 100644 --- a/src/sqlancer/doris/DorisSchema.java +++ b/src/sqlancer/doris/DorisSchema.java @@ -569,7 +569,7 @@ public static DorisSchema fromConnection(SQLConnection con, String databaseName) continue; } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); DorisTable t = new DorisTable(tableName, databaseColumns, isView); for (DorisColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 8141220fa..e4b760221 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -231,7 +231,7 @@ public static DuckDBSchema fromConnection(SQLConnection con, String databaseName continue; // TODO: unexpected? } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); List indexes = getIndexes(con, tableName); DuckDBTable t = new DuckDBTable(tableName, databaseColumns, indexes, isView); for (DuckDBColumn c : databaseColumns) { diff --git a/src/sqlancer/hive/HiveSchema.java b/src/sqlancer/hive/HiveSchema.java index 822eea163..8733d5caa 100644 --- a/src/sqlancer/hive/HiveSchema.java +++ b/src/sqlancer/hive/HiveSchema.java @@ -59,7 +59,7 @@ public static HiveSchema fromConnection(SQLConnection con, String databaseName) List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); HiveTable t = new HiveTable(tableName, databaseColumns, isView); for (HiveColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/hsqldb/HSQLDBSchema.java b/src/sqlancer/hsqldb/HSQLDBSchema.java index e1e1cb94b..2d41df83f 100644 --- a/src/sqlancer/hsqldb/HSQLDBSchema.java +++ b/src/sqlancer/hsqldb/HSQLDBSchema.java @@ -29,7 +29,7 @@ public static HSQLDBSchema fromConnection(SQLConnection connection, String datab continue; // TODO: unexpected? } List databaseColumns = getTableColumns(connection, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); HSQLDBSchema.HSQLDBTable t = new HSQLDBSchema.HSQLDBTable(tableName, databaseColumns, isView); for (HSQLDBSchema.HSQLDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 20337fea7..2c4607418 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -250,7 +250,7 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa boolean isPartitioned = "p".equals(rs.getString("relkind")); // TODO: also check insertable // TODO: insert into view? - boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || + boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || // tableTypeStr.contains("LOCAL TEMPORARY") && // !isInsertable; PostgresTable.TableType tableType = getTableType(tableTypeSchema); diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java index 112da7ef2..439615950 100644 --- a/src/sqlancer/presto/PrestoSchema.java +++ b/src/sqlancer/presto/PrestoSchema.java @@ -27,7 +27,7 @@ public static PrestoSchema fromConnection(SQLConnection con, String databaseName List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, databaseName, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); PrestoTable t = new PrestoTable(tableName, databaseColumns, isView); for (PrestoColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/questdb/QuestDBSchema.java b/src/sqlancer/questdb/QuestDBSchema.java index 8253ec82c..55ee01aab 100644 --- a/src/sqlancer/questdb/QuestDBSchema.java +++ b/src/sqlancer/questdb/QuestDBSchema.java @@ -268,7 +268,7 @@ public static QuestDBSchema fromConnection(SQLConnection con, String databaseNam continue; // TODO: unexpected? } List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); QuestDBTable t = new QuestDBTable(tableName, databaseColumns, isView); for (QuestDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/spark/SparkSchema.java b/src/sqlancer/spark/SparkSchema.java index 8d988e31e..9b3666916 100644 --- a/src/sqlancer/spark/SparkSchema.java +++ b/src/sqlancer/spark/SparkSchema.java @@ -54,7 +54,7 @@ public static SparkSchema fromConnection(SQLConnection con, String databaseName) List tableNames = getTableNames(con); for (String tableName : tableNames) { List databaseColumns = getTableColumns(con, tableName); - boolean isView = tableName.toLowerCase().startsWith("v"); + boolean isView = matchesViewName(tableName); SparkTable t = new SparkTable(tableName, databaseColumns, isView); for (SparkColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index 32e00504b..f8439c734 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -313,7 +313,7 @@ public static TiDBSchema fromConnection(SQLConnection con, String databaseName) continue; } List indexes = getIndexes(con, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); TiDBTable t = new TiDBTable(tableName, databaseColumns, indexes, isView); for (TiDBColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java index 1534b9b56..41762d453 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLSchema.java +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -216,7 +216,7 @@ public static YCQLSchema fromConnection(SQLConnection con, String databaseName) continue; } List databaseColumns = getTableColumns(con, databaseName, tableName); - boolean isView = tableName.startsWith("v"); + boolean isView = matchesViewName(tableName); YCQLTable t = new YCQLTable(tableName, databaseColumns, isView); for (YCQLColumn c : databaseColumns) { c.setTable(t); diff --git a/src/sqlancer/yugabyte/ysql/YSQLSchema.java b/src/sqlancer/yugabyte/ysql/YSQLSchema.java index 400a34e21..c75322af9 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLSchema.java +++ b/src/sqlancer/yugabyte/ysql/YSQLSchema.java @@ -81,7 +81,7 @@ public static YSQLSchema fromConnection(SQLConnection con, String databaseName) boolean isInsertable = rs.getBoolean("is_insertable_into"); // TODO: also check insertable // TODO: insert into view? - boolean isView = tableName.startsWith("v"); // tableTypeStr.contains("VIEW") || + boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || // tableTypeStr.contains("LOCAL TEMPORARY") && // !isInsertable; YSQLTable.TableType tableType = getTableType(tableTypeSchema); From 5cb7193514b16a3668cb6b3bfdbb17ee8d8e384f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Apr 2026 00:11:55 +0800 Subject: [PATCH 764/856] Use getFreeViewName() in view generators instead of inline logic PostgresViewGenerator, MaterializeViewGenerator, and YSQLViewGenerator had their own inline "v" + i++ loops to find free view names. Replace with the existing AbstractSchema.getFreeViewName() which provides the same logic plus randomized start indices for better fuzzing variety. Co-Authored-By: Claude Opus 4.6 --- .../materialize/gen/MaterializeViewGenerator.java | 14 +++----------- .../postgres/gen/PostgresViewGenerator.java | 14 +++----------- .../yugabyte/ysql/gen/YSQLViewGenerator.java | 14 +++----------- 3 files changed, 9 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/materialize/gen/MaterializeViewGenerator.java b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java index c8cd0e93f..e3cb8ff29 100644 --- a/src/sqlancer/materialize/gen/MaterializeViewGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeViewGenerator.java @@ -30,19 +30,11 @@ public static SQLQueryAdapter create(MaterializeGlobalState globalState) { materialized = false; } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } diff --git a/src/sqlancer/postgres/gen/PostgresViewGenerator.java b/src/sqlancer/postgres/gen/PostgresViewGenerator.java index b0a2a8b9d..10992ece6 100644 --- a/src/sqlancer/postgres/gen/PostgresViewGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresViewGenerator.java @@ -35,19 +35,11 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { materialized = false; } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java index f70891cc2..3005d49f8 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLViewGenerator.java @@ -28,19 +28,11 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { } } sb.append(" VIEW "); - int i = 0; - String[] name = new String[1]; - while (true) { - name[0] = "v" + i++; - if (globalState.getSchema().getDatabaseTables().stream() - .noneMatch(tab -> tab.getName().contentEquals(name[0]))) { - break; - } - } - sb.append(name[0]); + String name = globalState.getSchema().getFreeViewName(); + sb.append(name); sb.append("("); int nrColumns = Randomly.smallNumber() + 1; - for (i = 0; i < nrColumns; i++) { + for (int i = 0; i < nrColumns; i++) { if (i != 0) { sb.append(", "); } From e6a46c5803e5021a6f34d35f9982719e09cc6178 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 9 Apr 2026 09:12:41 +0800 Subject: [PATCH 765/856] Fix formatting in PostgresSchema after matchesViewName refactoring Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/PostgresSchema.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 2c4607418..8abd286f0 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -251,8 +251,8 @@ public static PostgresSchema fromConnection(SQLConnection con, String databaseNa // TODO: also check insertable // TODO: insert into view? boolean isView = matchesViewName(tableName); // tableTypeStr.contains("VIEW") || - // tableTypeStr.contains("LOCAL TEMPORARY") && - // !isInsertable; + // tableTypeStr.contains("LOCAL TEMPORARY") && + // !isInsertable; PostgresTable.TableType tableType = getTableType(tableTypeSchema); List databaseColumns = getTableColumns(con, tableName); List indexes = getIndexes(con, tableName); From 01d848de3ce1681b2fcc6d7393087e03bf606461 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Apr 2026 22:13:08 +0800 Subject: [PATCH 766/856] CI: Run reducer tests in Misc Tests job instead of separate job Combine the standalone reducer job into the existing Misc Tests step to reduce CI overhead. The reducer tests are lightweight and don't need their own runner. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ed6769c34..d35bf55b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**,!sqlancer.reducer.**' test + run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - name: Set up Python uses: actions/setup-python@v4 with: @@ -711,21 +711,3 @@ jobs: DORIS_AVAILABLE=true mvn -Dtest=TestDorisNoREC test DORIS_AVAILABLE=true mvn -Dtest=TestDorisPQS test DORIS_AVAILABLE=true mvn -Dtest=TestDorisTLP test - - reducer: - name: Reducer Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - - name: Build - run: mvn -B package -DskipTests=true - - name: Run Tests - run: | - mvn -Dtest=TestStatementReducer test From 3cdb5c51ce006d3b72e324977c8c44cd51b46fa6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 10 Apr 2026 22:18:43 +0800 Subject: [PATCH 767/856] Pull up isPrimaryKey() and hasPrimaryKey() into abstract base classes Add isPrimaryKey() to AbstractTableColumn (default false) and hasPrimaryKey() to AbstractTable, replacing duplicate implementations in MySQL, OceanBase, and TiDB. Add @Override to all database-specific isPrimaryKey() methods. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/cockroachdb/CockroachDBSchema.java | 1 + src/sqlancer/common/schema/AbstractTable.java | 4 ++++ src/sqlancer/common/schema/AbstractTableColumn.java | 4 ++++ src/sqlancer/databend/DatabendSchema.java | 1 + src/sqlancer/duckdb/DuckDBSchema.java | 1 + src/sqlancer/mariadb/MariaDBSchema.java | 1 + src/sqlancer/mysql/MySQLSchema.java | 5 +---- src/sqlancer/oceanbase/OceanBaseSchema.java | 5 +---- src/sqlancer/presto/PrestoSchema.java | 1 + src/sqlancer/sqlite3/schema/SQLite3Schema.java | 1 + src/sqlancer/tidb/TiDBSchema.java | 5 +---- src/sqlancer/yugabyte/ycql/YCQLSchema.java | 1 + 12 files changed, 18 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index c0eeedd2b..cbde577cb 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -182,6 +182,7 @@ public CockroachDBColumn(String name, CockroachDBCompositeDataType columnType, b this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/common/schema/AbstractTable.java b/src/sqlancer/common/schema/AbstractTable.java index 89d568867..58154681c 100644 --- a/src/sqlancer/common/schema/AbstractTable.java +++ b/src/sqlancer/common/schema/AbstractTable.java @@ -95,6 +95,10 @@ public boolean isView() { return isView; } + public boolean hasPrimaryKey() { + return columns.stream().anyMatch(c -> c.isPrimaryKey()); + } + public String getFreeColumnName() { int i = 0; if (Randomly.getBooleanWithRatherLowProbability()) { diff --git a/src/sqlancer/common/schema/AbstractTableColumn.java b/src/sqlancer/common/schema/AbstractTableColumn.java index e519bea97..a2f5fb1b3 100644 --- a/src/sqlancer/common/schema/AbstractTableColumn.java +++ b/src/sqlancer/common/schema/AbstractTableColumn.java @@ -12,6 +12,10 @@ public AbstractTableColumn(String name, T table, U type) { this.type = type; } + public boolean isPrimaryKey() { + return false; + } + public String getName() { return name; } diff --git a/src/sqlancer/databend/DatabendSchema.java b/src/sqlancer/databend/DatabendSchema.java index f6bf30757..89738a1f3 100644 --- a/src/sqlancer/databend/DatabendSchema.java +++ b/src/sqlancer/databend/DatabendSchema.java @@ -148,6 +148,7 @@ public DatabendColumn(String name, DatabendCompositeDataType columnType, boolean this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index e4b760221..857b1e008 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -132,6 +132,7 @@ public DuckDBColumn(String name, DuckDBCompositeDataType columnType, boolean isP this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/mariadb/MariaDBSchema.java b/src/sqlancer/mariadb/MariaDBSchema.java index 808653b25..7f7656d76 100644 --- a/src/sqlancer/mariadb/MariaDBSchema.java +++ b/src/sqlancer/mariadb/MariaDBSchema.java @@ -51,6 +51,7 @@ public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 0384f34df..c8a30614f 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -76,6 +76,7 @@ public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -194,10 +195,6 @@ public MySQLEngine getEngine() { return engine; } - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - } public static final class MySQLIndex extends TableIndex { diff --git a/src/sqlancer/oceanbase/OceanBaseSchema.java b/src/sqlancer/oceanbase/OceanBaseSchema.java index 7de2457ee..7b5b5954f 100644 --- a/src/sqlancer/oceanbase/OceanBaseSchema.java +++ b/src/sqlancer/oceanbase/OceanBaseSchema.java @@ -81,6 +81,7 @@ public int getPrecision() { return precision; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -195,10 +196,6 @@ public OceanBaseTable(String tableName, List columns, List c.isPrimaryKey()); - } - } public static final class OceanBaseIndex extends TableIndex { diff --git a/src/sqlancer/presto/PrestoSchema.java b/src/sqlancer/presto/PrestoSchema.java index 439615950..2e668969d 100644 --- a/src/sqlancer/presto/PrestoSchema.java +++ b/src/sqlancer/presto/PrestoSchema.java @@ -453,6 +453,7 @@ public PrestoColumn(String name, PrestoCompositeDataType columnType, boolean isP this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/sqlite3/schema/SQLite3Schema.java b/src/sqlancer/sqlite3/schema/SQLite3Schema.java index 586d7e776..fc97929d3 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3Schema.java +++ b/src/sqlancer/sqlite3/schema/SQLite3Schema.java @@ -78,6 +78,7 @@ public SQLite3Column(String rowId, SQLite3DataType columnType, boolean isInteger this.generated = generated; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } diff --git a/src/sqlancer/tidb/TiDBSchema.java b/src/sqlancer/tidb/TiDBSchema.java index f8439c734..4ce7306e5 100644 --- a/src/sqlancer/tidb/TiDBSchema.java +++ b/src/sqlancer/tidb/TiDBSchema.java @@ -170,6 +170,7 @@ public TiDBColumn(String name, TiDBCompositeDataType columnType, boolean isPrima this.hasDefault = hasDefault; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } @@ -297,10 +298,6 @@ public TiDBTable(String tableName, List columns, List in super(tableName, columns, indexes, isView); } - public boolean hasPrimaryKey() { - return getColumns().stream().anyMatch(c -> c.isPrimaryKey()); - } - } public static TiDBSchema fromConnection(SQLConnection con, String databaseName) throws SQLException { diff --git a/src/sqlancer/yugabyte/ycql/YCQLSchema.java b/src/sqlancer/yugabyte/ycql/YCQLSchema.java index 41762d453..736247364 100644 --- a/src/sqlancer/yugabyte/ycql/YCQLSchema.java +++ b/src/sqlancer/yugabyte/ycql/YCQLSchema.java @@ -126,6 +126,7 @@ public YCQLColumn(String name, YCQLCompositeDataType columnType, boolean isPrima this.isNullable = isNullable; } + @Override public boolean isPrimaryKey() { return isPrimaryKey; } From d13a1e4fc10956b14d6a580ee4aa99e876a6cd5c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Apr 2026 22:44:09 +0800 Subject: [PATCH 768/856] PostgreSQL: expect to_char "EEEE must be the last pattern used" error Random format strings fed to to_char (e.g. via md5(...)) can contain "eeee" followed by other characters, which PostgreSQL rejects with ERROR: "EEEE" must be the last pattern used. Add this to the known to_char expected errors so NoREC and similar oracles do not trip on it. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index e9f27aeb2..eeb160a56 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -122,6 +122,7 @@ private static List getToCharFunctionErrors() { errors.add("cannot use \"S\" and \"PL\" together"); errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); + errors.add("\"EEEE\" must be the last pattern used"); return errors; } From c9bc432f20d999edd73a9e9c3fe393c22a6ff2ee Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 12 Apr 2026 23:40:04 +0800 Subject: [PATCH 769/856] CI: upgrade Citus from PostgreSQL 15 + Citus 11.1 to PostgreSQL 17 + Citus 13.0 ubuntu-latest is now Ubuntu 24.04 (Noble), which has no Citus packages. Patch citusdata_community.list to use jammy after the curl installer creates it, then re-run apt-get update before installing. Fixes: https://github.com/citusdata/citus/issues/7692 Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d35bf55b0..45055c819 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -59,9 +59,11 @@ jobs: run: | echo "deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list curl https://install.citusdata.com/community/deb.sh | sudo bash - sudo apt-get -y install postgresql-15-citus-11.1 + sudo sed -i 's/noble/jammy/g' /etc/apt/sources.list.d/citusdata_community.list # https://github.com/citusdata/citus/issues/7692 + sudo apt-get update + sudo apt-get -y install postgresql-17-citus-13.0 sudo chown -R $USER:$USER /var/run/postgresql - export PATH=/usr/lib/postgresql/15/bin:$PATH + export PATH=/usr/lib/postgresql/17/bin:$PATH cd ~ mkdir -p citus/coordinator citus/worker1 citus/worker2 initdb -D citus/coordinator From 9780e5f58715421f989e0885966da30b08838df5 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Apr 2026 00:24:20 +0800 Subject: [PATCH 770/856] Fix Citus CI: handle reg* OID types in PostgresSchema PostgreSQL 17 + Citus 13.0 system tables expose columns with OID alias types (regnamespace, regrole, regtype, regproc, etc.) that were not handled by getColumnType(), causing an AssertionError. Map them to TEXT since they are textually representable, like the existing regclass entry. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/postgres/PostgresSchema.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/postgres/PostgresSchema.java b/src/sqlancer/postgres/PostgresSchema.java index 8abd286f0..82937557c 100644 --- a/src/sqlancer/postgres/PostgresSchema.java +++ b/src/sqlancer/postgres/PostgresSchema.java @@ -125,6 +125,12 @@ public static PostgresDataType getColumnType(String typeString) { case "character varying": case "name": case "regclass": + case "regnamespace": + case "regrole": + case "regtype": + case "regproc": + case "regprocedure": + case "regoper": return PostgresDataType.TEXT; case "numeric": return PostgresDataType.DECIMAL; From a2019cc2f2358cbeb769024ffe081d5b98f99e38 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Apr 2026 00:25:27 +0800 Subject: [PATCH 771/856] Fix Citus CI: filter citus_* system views and update error strings for Citus 13.0 - CitusSchema.fromConnection now skips all tables/views starting with "citus_" (not just "citus_tables") to handle the citus_schemas view added in Citus 12.0, which was causing unexpected SQL errors when SQLancer tried to DELETE/ALTER it after the regnamespace fix exposed it. - Update columnar table index error string from the old (incorrect) "indexes not supported for columnar tables" to the actual Citus error "unsupported access method for the index on columnar table", and add "BRIN indexes on columnar tables are not supported". - Add "alter table command is currently unsupported" for the Citus restriction on certain ALTER TABLE operations (e.g., multi-subcommand ALTERs, type changes) on distributed tables. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/citus/CitusSchema.java | 4 ++-- src/sqlancer/citus/gen/CitusCommon.java | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/citus/CitusSchema.java b/src/sqlancer/citus/CitusSchema.java index 560191c05..b2550bdce 100644 --- a/src/sqlancer/citus/CitusSchema.java +++ b/src/sqlancer/citus/CitusSchema.java @@ -63,8 +63,8 @@ public static CitusSchema fromConnection(SQLConnection con, String databaseName) "SELECT table_name, column_to_column_name(logicalrelid, partkey) AS dist_col_name, colocationid FROM information_schema.tables LEFT OUTER JOIN pg_dist_partition ON logicalrelid=table_name::regclass WHERE table_schema='public' OR table_schema LIKE 'pg_temp_%';")) { while (rs.next()) { String tableName = rs.getString("table_name"); - /* citus_tables is a helper view, we don't need to test with it so we let's ignore it */ - if (tableName.equals("citus_tables")) { + /* skip Citus-managed views in the public schema (citus_tables, citus_schemas, etc.) */ + if (tableName.startsWith("citus_")) { continue; } String distributionColumnName = rs.getString("dist_col_name"); diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 7a9f9c659..35aee85fa 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -23,10 +23,10 @@ public static List getCitusErrors() { errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); errors.add("functions used in UPDATE queries on distributed tables must not be VOLATILE"); errors.add("STABLE functions used in UPDATE queries cannot be called with column references"); - errors.add( - "functions used in the WHERE clause of modification queries on distributed tables must not be VOLATILE"); + errors.add("of modification queries on distributed tables must not be VOLATILE"); errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); errors.add("cannot execute ALTER TABLE command involving partition column"); + errors.add("alter table command is currently unsupported"); errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); errors.add("is not a regular, foreign or partitioned table"); errors.add("must be a distributed table or a reference table"); @@ -53,7 +53,8 @@ public static List getCitusErrors() { errors.add("direct joins between distributed and local tables are not supported"); errors.add("unlogged columnar tables are not supported"); errors.add("UPDATE and CTID scans not supported for ColumnarScan"); - errors.add("indexes not supported for columnar tables"); + errors.add("unsupported access method for the index on columnar table"); + errors.add("BRIN indexes on columnar tables are not supported"); errors.add("invalid byte sequence for encoding \"UTF8\": 0x00"); errors.add("columnar_tuple_insert_speculative not implemented"); errors.add("row field count is 1, expected 2"); From aed787157599f706c4d0d705d01b1b2fa7299efa Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 14 Apr 2026 00:25:51 +0800 Subject: [PATCH 772/856] Fix Citus CI: handle OID 0 errors for columnar temporary tables Citus 13.0 raises "could not open relation with OID 0" when operating on columnar temporary tables (USING columnar ON COMMIT DROP) during VACUUM, DISCARD TEMPORARY, and some INSERT operations where Citus cannot resolve the relation OID. Collect the error into getColumnarOidErrors() and wire it through Citus-specific generator wrappers: - CitusVacuumGenerator wraps PostgresVacuumGenerator so VACUUM accepts the OID 0 error (the VACUUM action previously used the Postgres generator directly). - CitusDiscardGenerator does the same for DISCARD. - INSERT already routes through CitusInsertGenerator, which picks up the error via addCitusErrors() in getCitusErrors(). Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/citus/CitusProvider.java | 16 +++++++-------- src/sqlancer/citus/gen/CitusCommon.java | 15 ++++++++++++++ .../citus/gen/CitusDiscardGenerator.java | 20 +++++++++++++++++++ .../citus/gen/CitusReindexGenerator.java | 20 +++++++++++++++++++ .../citus/gen/CitusTruncateGenerator.java | 20 +++++++++++++++++++ .../citus/gen/CitusVacuumGenerator.java | 20 +++++++++++++++++++ 6 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/citus/gen/CitusDiscardGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusReindexGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusTruncateGenerator.java create mode 100644 src/sqlancer/citus/gen/CitusVacuumGenerator.java diff --git a/src/sqlancer/citus/CitusProvider.java b/src/sqlancer/citus/CitusProvider.java index 747f72216..a11424f18 100644 --- a/src/sqlancer/citus/CitusProvider.java +++ b/src/sqlancer/citus/CitusProvider.java @@ -21,11 +21,15 @@ import sqlancer.citus.gen.CitusAlterTableGenerator; import sqlancer.citus.gen.CitusCommon; import sqlancer.citus.gen.CitusDeleteGenerator; +import sqlancer.citus.gen.CitusDiscardGenerator; import sqlancer.citus.gen.CitusIndexGenerator; import sqlancer.citus.gen.CitusInsertGenerator; +import sqlancer.citus.gen.CitusReindexGenerator; import sqlancer.citus.gen.CitusSetGenerator; import sqlancer.citus.gen.CitusTableGenerator; +import sqlancer.citus.gen.CitusTruncateGenerator; import sqlancer.citus.gen.CitusUpdateGenerator; +import sqlancer.citus.gen.CitusVacuumGenerator; import sqlancer.citus.gen.CitusViewGenerator; import sqlancer.common.DBMSCommon; import sqlancer.common.oracle.CompositeTestOracle; @@ -44,15 +48,11 @@ import sqlancer.postgres.gen.PostgresAnalyzeGenerator; import sqlancer.postgres.gen.PostgresClusterGenerator; import sqlancer.postgres.gen.PostgresCommentGenerator; -import sqlancer.postgres.gen.PostgresDiscardGenerator; import sqlancer.postgres.gen.PostgresDropIndexGenerator; import sqlancer.postgres.gen.PostgresNotifyGenerator; -import sqlancer.postgres.gen.PostgresReindexGenerator; import sqlancer.postgres.gen.PostgresSequenceGenerator; import sqlancer.postgres.gen.PostgresStatisticsGenerator; import sqlancer.postgres.gen.PostgresTransactionGenerator; -import sqlancer.postgres.gen.PostgresTruncateGenerator; -import sqlancer.postgres.gen.PostgresVacuumGenerator; @AutoService(DatabaseProvider.class) public class CitusProvider extends PostgresProvider { @@ -82,13 +82,13 @@ public enum Action implements AbstractAction { CREATE_STATISTICS(PostgresStatisticsGenerator::insert), // DROP_STATISTICS(PostgresStatisticsGenerator::remove), // DELETE(CitusDeleteGenerator::create), // - DISCARD(PostgresDiscardGenerator::create), // + DISCARD(CitusDiscardGenerator::create), // DROP_INDEX(PostgresDropIndexGenerator::create), // INSERT(CitusInsertGenerator::insert), // UPDATE(CitusUpdateGenerator::create), // - TRUNCATE(PostgresTruncateGenerator::create), // - VACUUM(PostgresVacuumGenerator::create), // - REINDEX(PostgresReindexGenerator::create), // + TRUNCATE(CitusTruncateGenerator::create), // + VACUUM(CitusVacuumGenerator::create), // + REINDEX(CitusReindexGenerator::create), // SET(CitusSetGenerator::create), // CREATE_INDEX(CitusIndexGenerator::generate), // SET_CONSTRAINTS((g) -> { diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index 35aee85fa..ba8936aaa 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -27,6 +27,7 @@ public static List getCitusErrors() { errors.add("cannot execute ADD CONSTRAINT command with other subcommands"); errors.add("cannot execute ALTER TABLE command involving partition column"); errors.add("alter table command is currently unsupported"); + errors.add("on distributed partitioned tables are not supported"); errors.add("could not run distributed query with FOR UPDATE/SHARE commands"); errors.add("is not a regular, foreign or partitioned table"); errors.add("must be a distributed table or a reference table"); @@ -61,6 +62,7 @@ public static List getCitusErrors() { errors.add("incorrect binary data format"); errors.add("invalid sign in external \"numeric\" value"); errors.add("Foreign keys and AFTER ROW triggers are not supported for columnar tables"); + errors.addAll(getColumnarOidErrors()); // current errors in Citus (to be removed once fixed) if (CitusBugs.bug3957) { @@ -88,6 +90,19 @@ public static List getCitusErrors() { return errors; } + /** + * Citus can fail with "could not open relation with OID 0" when operating on columnar temporary tables (e.g., USING + * columnar ON COMMIT DROP), during VACUUM, DISCARD TEMPORARY, or INSERT operations where Citus cannot resolve the + * relation OID. + * + * @return the list of expected error substrings for columnar OID resolution failures. + */ + public static List getColumnarOidErrors() { + List errors = new ArrayList<>(); + errors.add("could not open relation with OID 0"); + return errors; + } + public static void addCitusErrors(ExpectedErrors errors) { errors.addAll(getCitusErrors()); } diff --git a/src/sqlancer/citus/gen/CitusDiscardGenerator.java b/src/sqlancer/citus/gen/CitusDiscardGenerator.java new file mode 100644 index 000000000..f4a1b3240 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusDiscardGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresDiscardGenerator; + +public final class CitusDiscardGenerator { + + private CitusDiscardGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter discardQuery = PostgresDiscardGenerator.create(globalState); + ExpectedErrors errors = discardQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return discardQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusReindexGenerator.java b/src/sqlancer/citus/gen/CitusReindexGenerator.java new file mode 100644 index 000000000..6f37cbe06 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusReindexGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresReindexGenerator; + +public final class CitusReindexGenerator { + + private CitusReindexGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter reindexQuery = PostgresReindexGenerator.create(globalState); + ExpectedErrors errors = reindexQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return reindexQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusTruncateGenerator.java b/src/sqlancer/citus/gen/CitusTruncateGenerator.java new file mode 100644 index 000000000..cf36ce9c2 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusTruncateGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresTruncateGenerator; + +public final class CitusTruncateGenerator { + + private CitusTruncateGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter truncateQuery = PostgresTruncateGenerator.create(globalState); + ExpectedErrors errors = truncateQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return truncateQuery; + } + +} diff --git a/src/sqlancer/citus/gen/CitusVacuumGenerator.java b/src/sqlancer/citus/gen/CitusVacuumGenerator.java new file mode 100644 index 000000000..ae73dbf82 --- /dev/null +++ b/src/sqlancer/citus/gen/CitusVacuumGenerator.java @@ -0,0 +1,20 @@ +package sqlancer.citus.gen; + +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.gen.PostgresVacuumGenerator; + +public final class CitusVacuumGenerator { + + private CitusVacuumGenerator() { + } + + public static SQLQueryAdapter create(PostgresGlobalState globalState) { + SQLQueryAdapter vacuumQuery = PostgresVacuumGenerator.create(globalState); + ExpectedErrors errors = vacuumQuery.getExpectedErrors(); + CitusCommon.addCitusErrors(errors); + return vacuumQuery; + } + +} From f88317414049f709632a88efe442d660bc7f7002 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 15 Apr 2026 09:49:49 +0800 Subject: [PATCH 773/856] CI: skip JaCoCo report for test runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The JaCoCo report goal intermittently fails with EOFException when the forked test JVM is force-killed before the agent can finish flushing jacoco.exec. This has been especially visible on QPG/Materialize and other remote-DBMS jobs where long-lived worker threads continue past test completion. The report isn't uploaded or consumed anywhere in CI — it's generated and discarded — so skip it with -Djacoco.skip=true. Developers can still produce coverage locally by running mvn test without the flag. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 108 ++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45055c819..40306d847 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: - name: Verify run: mvn -B verify -DskipTests=true - name: Misc Tests - run: mvn -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test + run: mvn -Djacoco.skip=true -B '-Dtest=!sqlancer.dbms.**,!sqlancer.qpg.**' test - name: Set up Python uses: actions/setup-python@v4 with: @@ -88,7 +88,7 @@ jobs: psql -c "SELECT * from citus_add_node('localhost', 9701);" -p 9700 -U $USER -d test psql -c "SELECT * from citus_add_node('localhost', 9702);" -p 9700 -U $USER -d test - name: Run Tests - run: CITUS_AVAILABLE=true mvn -Dtest=TestCitus test + run: CITUS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCitus test cnosdb: name: DBMS Tests (CnosDB, creation only) @@ -110,9 +110,9 @@ jobs: until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done - name: Run Tests run: | - CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBNoREC test + CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBNoREC test sleep 20 - CNOSDB_AVAILABLE=true mvn -Dtest=TestCnosDBTLP test + CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBTLP test clickhouse: name: DBMS Tests (ClickHouse) @@ -133,7 +133,7 @@ jobs: docker run --ulimit nofile=262144:262144 --name clickhouse-server -p8123:8123 -d clickhouse/clickhouse-server:24.3.1.2672 until curl -sf http://127.0.0.1:8123/ping 2>/dev/null; do sleep 1; done - name: Run Tests - run: CLICKHOUSE_AVAILABLE=true mvn -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test + run: CLICKHOUSE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=ClickHouseBinaryComparisonOperationTest,TestClickHouse,ClickHouseOperatorsVisitorTest,ClickHouseToStringVisitorTest test - name: Show fatal errors run: docker exec clickhouse-server grep Fatal /var/log/clickhouse-server/clickhouse-server.log || echo No Fatal Errors found - name: Teardown ClickHouse server @@ -163,9 +163,9 @@ jobs: run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests run: | - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBNoREC test - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBTLP test - COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBCERT test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBNoREC test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBTLP test + COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBCERT test cockroachdb-qpg: name: QPG Tests (CockroachDB) @@ -188,7 +188,7 @@ jobs: - name: Create SQLancer user run: cd cockroach-v24.2.0.linux-amd64/ && ./cockroach sql --insecure -e "CREATE USER sqlancer; GRANT admin to sqlancer" && cd .. - name: Run Tests - run: COCKROACHDB_AVAILABLE=true mvn -Dtest=TestCockroachDBQPG test + run: COCKROACHDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCockroachDBQPG test databend: name: DBMS Tests (Databend) @@ -214,9 +214,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendTLP test - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendNoREC test - DATABEND_AVAILABLE=true mvn -Dtest=TestDatabendPQS test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendTLP test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendNoREC test + DATABEND_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDatabendPQS test datafusion: name: DBMS Tests (DataFusion) @@ -258,7 +258,7 @@ jobs: exit 1 - name: Run Tests run: | - DATAFUSION_AVAILABLE=true mvn test -Pdatafusion-tests + DATAFUSION_AVAILABLE=true mvn -Djacoco.skip=true test -Pdatafusion-tests duckdb: name: DBMS Tests (DuckDB) @@ -276,8 +276,8 @@ jobs: run: mvn -B package -DskipTests=true - name: DuckDB Tests run: | - mvn -Dtest=TestDuckDBTLP test - mvn -Dtest=TestDuckDBNoREC test + mvn -Djacoco.skip=true -Dtest=TestDuckDBTLP test + mvn -Djacoco.skip=true -Dtest=TestDuckDBNoREC test h2: name: DBMS Tests (H2) @@ -293,7 +293,7 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: mvn -Dtest=TestH2 test + run: mvn -Djacoco.skip=true -Dtest=TestH2 test hive: name: DBMS Tests (Hive) @@ -327,7 +327,7 @@ jobs: - name: Build SQLancer run: mvn -B package -DskipTests=true - name: Run Tests - run: HIVE_AVAILABLE=true mvn -Dtest=TestHiveTLP test + run: HIVE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestHiveTLP test spark: name: DBMS Tests (Spark) @@ -365,7 +365,7 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests - run: SPARK_AVAILABLE=true mvn -Dtest=TestSparkTLP test + run: SPARK_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestSparkTLP test hsqldb: name: DBMS Tests (HSQLDB) @@ -382,8 +382,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - mvn -Dtest=TestHSQLDBNoREC test - mvn -Dtest=TestHSQLDBTLP test + mvn -Djacoco.skip=true -Dtest=TestHSQLDBNoREC test + mvn -Djacoco.skip=true -Dtest=TestHSQLDBTLP test mariadb: name: DBMS Tests (MariaDB) @@ -410,7 +410,7 @@ jobs: - name: Create SQLancer User run: sudo mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests - run: MARIADB_AVAILABLE=true mvn -Dtest=TestMariaDB test + run: MARIADB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestMariaDB test materialize: name: DBMS Tests (Materialize) @@ -432,9 +432,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeNoREC - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeTLP - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializePQS + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeNoREC + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeTLP + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializePQS materialize-qpg: name: QPG Tests (Materialize) @@ -456,8 +456,8 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQPG - MATERIALIZE_AVAILABLE=true mvn test -Dtest=TestMaterializeQueryPlan + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeQPG + MATERIALIZE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMaterializeQueryPlan mysql: name: DBMS Tests (MySQL, CERT creation only) @@ -484,10 +484,10 @@ jobs: run: mysql -h 127.0.0.1 -uroot -proot -e "CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLPQS - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLTLP - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLCERT - MYSQL_AVAILABLE=true mvn test -Dtest=TestMySQLDQE + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLPQS + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLTLP + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLCERT + MYSQL_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestMySQLDQE oceanbase: name: DBMS Tests (OceanBase) @@ -509,9 +509,9 @@ jobs: mysql -h127.1 -uroot@test -P2881 -Doceanbase -A -e"CREATE USER 'sqlancer'@'%' IDENTIFIED BY 'sqlancer'; GRANT ALL PRIVILEGES ON * . * TO 'sqlancer'@'%';" - name: Run Tests run: | - OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseNoREC - OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBasePQS - OCEANBASE_AVAILABLE=true mvn test -Dtest=TestOceanBaseTLP + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBaseNoREC + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBasePQS + OCEANBASE_AVAILABLE=true mvn -Djacoco.skip=true test -Dtest=TestOceanBaseTLP postgres: name: DBMS Tests (PostgreSQL) runs-on: ubuntu-latest @@ -534,10 +534,10 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresPQS test - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresTLP test - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresNoREC test - POSTGRES_AVAILABLE=true mvn -Dtest=TestPostgresCERT test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresPQS test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresTLP test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresNoREC test + POSTGRES_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPostgresCERT test presto: name: DBMS Tests (Presto) @@ -560,9 +560,9 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoNoREC test + PRESTO_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPrestoNoREC test docker restart presto && until curl -sf http://127.0.0.1:8080/v1/info 2>/dev/null; do sleep 2; done - PRESTO_AVAILABLE=true mvn -Dtest=TestPrestoTLP test + PRESTO_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestPrestoTLP test sqlite: name: DBMS Tests (SQLite) runs-on: ubuntu-latest @@ -579,10 +579,10 @@ jobs: run: mvn -B package -DskipTests=true - name: SQLite Tests run: | - mvn -Dtest=TestSQLitePQS test - mvn -Dtest=TestSQLiteTLP test - mvn -Dtest=TestSQLiteNoREC test - mvn -Dtest=TestSQLiteCODDTest test + mvn -Djacoco.skip=true -Dtest=TestSQLitePQS test + mvn -Djacoco.skip=true -Dtest=TestSQLiteTLP test + mvn -Djacoco.skip=true -Dtest=TestSQLiteNoREC test + mvn -Djacoco.skip=true -Dtest=TestSQLiteCODDTest test sqlite-qpg: name: QPG Tests (SQLite) @@ -600,7 +600,7 @@ jobs: run: mvn -B package -DskipTests=true - name: SQLite Tests for QPG run: | - mvn -Dtest=TestSQLiteQPG test + mvn -Djacoco.skip=true -Dtest=TestSQLiteQPG test tidb: name: DBMS Tests (TiDB, TLP creation only) @@ -624,8 +624,8 @@ jobs: run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests run: | - TIDB_AVAILABLE=true mvn -Dtest=TestTiDBTLP test - TIDB_AVAILABLE=true mvn -Dtest=TestTiDBCERT test + TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBTLP test + TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBCERT test tidb-qpg: name: QPG Tests (TiDB) @@ -648,7 +648,7 @@ jobs: - name: Create SQLancer user run: mysql -h 127.0.0.1 -P 4000 -u root -D test -e "CREATE USER 'sqlancer'@'%' IDENTIFIED WITH mysql_native_password BY 'sqlancer'; GRANT ALL PRIVILEGES ON *.* TO 'sqlancer'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;" - name: Run Tests - run: TIDB_AVAILABLE=true mvn -Dtest=TestTiDBQPG test + run: TIDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestTiDBQPG test yugabyte: name: DBMS Tests (YugabyteDB) @@ -670,10 +670,10 @@ jobs: until pg_isready -h localhost -p 5433; do sleep 1; done - name: Run Tests run: | - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLNoREC test - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLTLP test - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYSQLPQS test - YUGABYTE_AVAILABLE=true mvn -Dtest=TestYCQL test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLNoREC test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLTLP test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLPQS test + YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYCQL test doris: name: DBMS Tests (Apache Doris) @@ -710,6 +710,6 @@ jobs: run: mvn -B package -DskipTests=true - name: Run Tests run: | - DORIS_AVAILABLE=true mvn -Dtest=TestDorisNoREC test - DORIS_AVAILABLE=true mvn -Dtest=TestDorisPQS test - DORIS_AVAILABLE=true mvn -Dtest=TestDorisTLP test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisNoREC test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisPQS test + DORIS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestDorisTLP test From 8804b3f1d627426a64db40ca5d221e04d91872b3 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 16 Apr 2026 22:38:19 +0800 Subject: [PATCH 774/856] Fix YugabyteDB CI: use DROP DATABASE WITH (FORCE) DROP DATABASE could fail with "database is being accessed by other users" when a prior iteration's session had not yet been released, causing tests to exit with -1. Adding WITH (FORCE) terminates lingering sessions as part of the drop. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/yugabyte/ysql/YSQLProvider.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/yugabyte/ysql/YSQLProvider.java b/src/sqlancer/yugabyte/ysql/YSQLProvider.java index e72790344..efcc7ec22 100644 --- a/src/sqlancer/yugabyte/ysql/YSQLProvider.java +++ b/src/sqlancer/yugabyte/ysql/YSQLProvider.java @@ -204,11 +204,11 @@ private void createDatabaseSync(YSQLGlobalState globalState, String entryDatabas Connection con = createConnectionSafely(entryURL, username, password); globalState.getState().logStatement(String.format("\\c %s;", entryDatabaseName)); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); + globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)"); createDatabaseCommand = getCreateDatabaseCommand(globalState); globalState.getState().logStatement(createDatabaseCommand); try (Statement s = con.createStatement()) { - s.execute("DROP DATABASE IF EXISTS " + databaseName); + s.execute("DROP DATABASE IF EXISTS " + databaseName + " WITH (FORCE)"); } try (Statement s = con.createStatement()) { s.execute(createDatabaseCommand); From 0c477bbfb553ce4e158f4f49b516e990f7b10b68 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 16 Apr 2026 23:16:43 +0800 Subject: [PATCH 775/856] Fix Citus CI: add distribution-column filter error for recursive CTEs as expected error Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/citus/gen/CitusCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/citus/gen/CitusCommon.java b/src/sqlancer/citus/gen/CitusCommon.java index ba8936aaa..58b1b7c16 100644 --- a/src/sqlancer/citus/gen/CitusCommon.java +++ b/src/sqlancer/citus/gen/CitusCommon.java @@ -18,6 +18,7 @@ public static List getCitusErrors() { errors.add("cannot perform an INSERT without a partition column value"); errors.add("cannot perform an INSERT with NULL in the partition column"); errors.add("recursive CTEs are not supported in distributed queries"); + errors.add("recursive CTEs are only supported when they contain a filter on the distribution column"); errors.add("could not run distributed query with GROUPING SETS, CUBE, or ROLLUP"); errors.add("Subqueries in HAVING cannot refer to outer query"); errors.add("non-IMMUTABLE functions are not allowed in the RETURNING clause"); From e44eba43304e7ea0d4f65c639fa9a13f06bfc5d9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 17 Apr 2026 23:06:03 +0800 Subject: [PATCH 776/856] Fix Databend CI: upgrade to v1.2.896-nightly and handle bug #19738 The old Databend image (v1.2.687-nightly) had a server hang triggered by SELECT DISTINCT with large LIMIT values, which caused PQS tests to fail on every CI run. Upgrade to v1.2.896-nightly (which fixes the hang) and work around https://github.com/databendlabs/databend/issues/19738, where SELECT AVG(constant) over a cross join causes an internal assertion failure (UnwindError with Decimal precision mismatch). Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/DatabendBugs.java | 1 + src/sqlancer/databend/DatabendErrors.java | 4 ++++ .../tlp/DatabendQueryPartitioningAggregateTester.java | 11 ++++++++--- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 40306d847..ee4186650 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.687-nightly + image: datafuselabs/databend:v1.2.896-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index dd11512d8..a058ccc32 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -19,6 +19,7 @@ public final class DatabendBugs { public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 + public static boolean bug19738 = true; // https://github.com/databendlabs/databend/issues/19738 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index fdd8a3a69..746a4e848 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -47,6 +47,10 @@ public static List getExpressionErrors() { if (DatabendBugs.bug15568) { errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); } + if (DatabendBugs.bug19738) { + errors.add("UnwindError"); + errors.add("unable to cast `NULL`"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 0ba0d9c86..6d52caea3 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -10,6 +10,7 @@ import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -44,9 +45,13 @@ public DatabendQueryPartitioningAggregateTester(DatabendGlobalState state) { @Override public void check() throws SQLException { super.check(); - DatabendAggregateFunction aggregateFunction = Randomly.fromOptions(DatabendAggregateFunction.MAX, - DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT, - DatabendAggregateFunction.AVG/* , DatabendAggregateFunction.STDDEV_POP */); + List aggregateFunctions = new ArrayList<>(List.of(DatabendAggregateFunction.MAX, + DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT + /* , DatabendAggregateFunction.STDDEV_POP */)); + if (!DatabendBugs.bug19738) { + aggregateFunctions.add(DatabendAggregateFunction.AVG); + } + DatabendAggregateFunction aggregateFunction = Randomly.fromList(aggregateFunctions); DatabendFunctionOperation aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); List fetchColumns = new ArrayList<>(); From d6533d2e4c4453ce8947f3f9420862e5892e0750 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 00:07:58 +0800 Subject: [PATCH 777/856] Fix Presto CI: handle bytecode compiler bugs #27608 and #27609 Walk the exception cause chain in ComparatorHelper (consistent with SQLQueryAdapter.checkException) so wrapped errors like the JDBC driver's RuntimeException are matched against expected errors. Add VerifyError (#27608) and Compiler failed (#27609) as expected Presto bugs triggered by complex CASE expressions on Presto 0.297. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/ComparatorHelper.java | 11 ++++++----- src/sqlancer/presto/PrestoBugs.java | 6 ++++++ src/sqlancer/presto/PrestoErrors.java | 7 +++++++ 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/ComparatorHelper.java b/src/sqlancer/ComparatorHelper.java index 5da635de2..cee290924 100644 --- a/src/sqlancer/ComparatorHelper.java +++ b/src/sqlancer/ComparatorHelper.java @@ -70,11 +70,12 @@ public static List getResultSetFirstColumnAsString(String queryString, E throw e; } - if (e.getMessage() == null) { - throw new AssertionError(queryString, e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); + Throwable current = e; + while (current != null) { + if (current.getMessage() != null && errors.errorIsExpected(current.getMessage())) { + throw new IgnoreMeException(); + } + current = current.getCause(); } throw new AssertionError(queryString, e); } finally { diff --git a/src/sqlancer/presto/PrestoBugs.java b/src/sqlancer/presto/PrestoBugs.java index f5e888df1..b0eb3fe57 100644 --- a/src/sqlancer/presto/PrestoBugs.java +++ b/src/sqlancer/presto/PrestoBugs.java @@ -8,6 +8,12 @@ public final class PrestoBugs { // https://github.com/prestodb/presto/issues/23613 public static boolean bug23613 = true; + // https://github.com/prestodb/presto/issues/27608 + public static boolean bugVerifyError = true; + + // https://github.com/prestodb/presto/issues/27609 + public static boolean bugCompilerFailed = true; + private PrestoBugs() { } diff --git a/src/sqlancer/presto/PrestoErrors.java b/src/sqlancer/presto/PrestoErrors.java index 2296223d6..dd2931976 100644 --- a/src/sqlancer/presto/PrestoErrors.java +++ b/src/sqlancer/presto/PrestoErrors.java @@ -47,6 +47,13 @@ public static List getExpressionErrors() { } errors.add("Cannot cast java.lang.String to java.util.List"); errors.add("Unexpected subquery expression in logical plan"); + if (PrestoBugs.bugVerifyError) { + errors.add("VerifyError"); + } + if (PrestoBugs.bugCompilerFailed) { + errors.add("Compiler failed"); + errors.add("Error processing class definition"); + } // 9223372036854775808 errors.add("Invalid numeric literal"); From e5ea4affea8271210966c04b89367e1f3d506b30 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 00:23:07 +0800 Subject: [PATCH 778/856] Fix Hive CI: skip unary prefix operators to work around negation nullability bug Hive incorrectly evaluates IS NULL for negated expressions involving string concatenation with column references (e.g., -(c || 'x') IS NULL returns false instead of true). The optimizer's nullability inference for GenericUDFOPNegative does not account for runtime NULL from non-null input. Affects Hive 4.0.1 and 4.2.0. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/hive/HiveBugs.java | 18 ++++++++++++++++++ .../hive/gen/HiveExpressionGenerator.java | 9 ++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 src/sqlancer/hive/HiveBugs.java diff --git a/src/sqlancer/hive/HiveBugs.java b/src/sqlancer/hive/HiveBugs.java new file mode 100644 index 000000000..01b772bb8 --- /dev/null +++ b/src/sqlancer/hive/HiveBugs.java @@ -0,0 +1,18 @@ +package sqlancer.hive; + +// do not make the fields final to avoid warnings +public final class HiveBugs { + + // Incorrect IS NULL evaluation for negation of string concatenation involving column references. + // -(c || 'x') evaluates to NULL at runtime, but IS NULL incorrectly returns false. + // The optimizer's nullability inference for GenericUDFOPNegative does not account for + // runtime conversion failures producing NULL from non-null input. + // Reproduce: CREATE TABLE t(c DOUBLE); INSERT INTO t VALUES(1.0); + // SELECT (-(c || 'x')) IS NULL FROM t; -- returns false, expected true + // Affects: 4.0.1, 4.2.0 + public static boolean bugNegationNullability = true; + + private HiveBugs() { + } + +} diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index 9f3de2514..a31420899 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -5,12 +5,14 @@ import java.util.List; import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.ast.BinaryOperatorNode.Operator; import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; +import sqlancer.hive.HiveBugs; import sqlancer.hive.HiveGlobalState; import sqlancer.hive.HiveSchema.HiveColumn; import sqlancer.hive.HiveSchema.HiveDataType; @@ -80,7 +82,12 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr Expression expr = Randomly.fromList(possibleOptions); switch (expr) { case UNARY_PREFIX: - return new HiveUnaryPrefixOperation(generateExpression(depth + 1), HiveUnaryPrefixOperator.getRandom()); + HiveUnaryPrefixOperator prefixOp = HiveUnaryPrefixOperator.getRandom(); + if (HiveBugs.bugNegationNullability + && (prefixOp == HiveUnaryPrefixOperator.MINUS || prefixOp == HiveUnaryPrefixOperator.PLUS)) { + throw new IgnoreMeException(); + } + return new HiveUnaryPrefixOperation(generateExpression(depth + 1), prefixOp); case UNARY_POSTFIX: return new HiveUnaryPostfixOperation(generateExpression(depth + 1), HiveUnaryPostfixOperator.getRandom()); case BINARY_COMPARISON: From 8ee570f530b01890d77e7f101ec34ee47904bef7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 00:44:50 +0800 Subject: [PATCH 779/856] Fix OceanBase CI: add "value is out of range" as expected expression error Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/oceanbase/OceanBaseErrors.java | 1 + src/sqlancer/oceanbase/OceanBaseOracleFactory.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/oceanbase/OceanBaseErrors.java b/src/sqlancer/oceanbase/OceanBaseErrors.java index c539ab129..7a5bd8a77 100644 --- a/src/sqlancer/oceanbase/OceanBaseErrors.java +++ b/src/sqlancer/oceanbase/OceanBaseErrors.java @@ -15,6 +15,7 @@ public static List getExpressionErrors() { ArrayList errors = new ArrayList<>(); errors.add("BIGINT value is out of range"); // e.g., CAST(-('-1e500') AS SIGNED) + errors.add("value is out of range"); errors.add("is not valid for CHARACTER SET"); errors.add("The observer or zone is not the master"); errors.add("Incorrect integer value"); diff --git a/src/sqlancer/oceanbase/OceanBaseOracleFactory.java b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java index b7b115f37..b1ab1cb5b 100644 --- a/src/sqlancer/oceanbase/OceanBaseOracleFactory.java +++ b/src/sqlancer/oceanbase/OceanBaseOracleFactory.java @@ -17,7 +17,7 @@ public enum OceanBaseOracleFactory implements OracleFactory create(OceanBaseGlobalState globalState) throws SQLException { OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState); ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(OceanBaseErrors.getExpressionErrors()) - .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).with("value is out of range").build(); + .withRegex(OceanBaseErrors.getExpressionErrorsRegex()).build(); return new TLPWhereOracle<>(globalState, gen, expectedErrors); } From 220137f84371f2ebf64c1649f1f741d96c7fb056 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 01:12:55 +0800 Subject: [PATCH 780/856] Fix Hive CI: skip additional expression types triggering Hive evaluation bugs Add bug flags for three more Hive bugs: - bugNonBooleanWhereClause: CAST/FLOOR/ROUND/arithmetic silently return 0 rows when used as WHERE predicates instead of erroring - bugInBooleanEvaluation: IN with boolean/IS NULL sub-expressions returns 0 rows for all TLP partitions - bugBetweenMixedTypes: BETWEEN with mixed boolean/numeric types loses rows in TLP partitions Skip CAST, FUNC, BINARY_ARITHMETIC, IN, and BETWEEN expression types in the generator when the corresponding bug flags are enabled. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/hive/HiveBugs.java | 23 +++++++++++++++++++ .../hive/gen/HiveExpressionGenerator.java | 12 +++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/hive/HiveBugs.java b/src/sqlancer/hive/HiveBugs.java index 01b772bb8..43d42ce0b 100644 --- a/src/sqlancer/hive/HiveBugs.java +++ b/src/sqlancer/hive/HiveBugs.java @@ -12,6 +12,29 @@ public final class HiveBugs { // Affects: 4.0.1, 4.2.0 public static boolean bugNegationNullability = true; + // Non-boolean expressions (CAST to non-boolean, FLOOR, ROUND, arithmetic) silently + // return 0 rows for all three TLP partitions when used as WHERE predicates. + // Hive requires BOOLEAN in WHERE but does not error; instead it returns empty results. + // Reproduce: CREATE TABLE t(c INT); INSERT INTO t VALUES(1); + // SELECT * FROM t WHERE FLOOR(1); -- returns 0 rows, expected 1 + // Affects: 4.0.1, 4.2.0 + public static boolean bugNonBooleanWhereClause = true; + + // IN operator with boolean sub-expressions involving IS NULL evaluates incorrectly, + // returning 0 rows for all three TLP partitions. + // Reproduce: CREATE TABLE t(c BOOLEAN); INSERT INTO t VALUES(true),(false); + // SELECT * FROM t WHERE (c != c) IN ((false) IS NULL); -- returns 0, expected 2 + // Affects: 4.0.1, 4.2.0 + public static boolean bugInBooleanEvaluation = true; + + // BETWEEN with mixed boolean/numeric types has incorrect TLP evaluation. + // The IS NULL partition misses rows due to wrong nullability inference. + // Reproduce: CREATE TABLE t(c DOUBLE); INSERT INTO t VALUES(0.5),(1.5); + // SELECT * FROM t WHERE (c NOT IN (true)) NOT BETWEEN 0.01 AND c; + // -- TLP partitions lose rows + // Affects: 4.0.1, 4.2.0 + public static boolean bugBetweenMixedTypes = true; + private HiveBugs() { } diff --git a/src/sqlancer/hive/gen/HiveExpressionGenerator.java b/src/sqlancer/hive/gen/HiveExpressionGenerator.java index a31420899..92154873c 100644 --- a/src/sqlancer/hive/gen/HiveExpressionGenerator.java +++ b/src/sqlancer/hive/gen/HiveExpressionGenerator.java @@ -77,7 +77,17 @@ private HiveExpression generateExpressionInternal(int depth) throws AssertionErr } List possibleOptions = new ArrayList<>(Arrays.asList(Expression.values())); - // TODO: remove some of the possible expression types according to options. + if (HiveBugs.bugNonBooleanWhereClause) { + possibleOptions.remove(Expression.CAST); + possibleOptions.remove(Expression.FUNC); + possibleOptions.remove(Expression.BINARY_ARITHMETIC); + } + if (HiveBugs.bugInBooleanEvaluation) { + possibleOptions.remove(Expression.IN); + } + if (HiveBugs.bugBetweenMixedTypes) { + possibleOptions.remove(Expression.BETWEEN); + } Expression expr = Randomly.fromList(possibleOptions); switch (expr) { From e81ffd1ddbf326f485a79ed9dd28906263fd7cb1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 19 Apr 2026 13:56:04 +0800 Subject: [PATCH 781/856] Fix Materialize CI: retry readSchema to handle eventual consistency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Materialize's information_schema is eventually consistent — tables and columns may not be visible immediately after creation. This caused IndexOutOfBoundsException when generators called getRandomTable() on an empty or incomplete schema. Retry readSchema() until the snapshot has no tables with empty columns, the table count has not regressed, and — after the initial read — the table list is not suspiciously empty. Co-Authored-By: Claude Opus 4.6 --- .../materialize/MaterializeGlobalState.java | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java index 77cbe5d14..8a66a7819 100644 --- a/src/sqlancer/materialize/MaterializeGlobalState.java +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -27,6 +27,8 @@ public class MaterializeGlobalState extends SQLGlobalState functionsAndTypes = new HashMap<>(); private List allowedFunctionTypes = Arrays.asList(IMMUTABLE, STABLE, VOLATILE); + private int lastKnownTableCount; + private int readSchemaCallCount; @Override public void setConnection(SQLConnection con) { @@ -266,7 +268,30 @@ public String getRandomTableAccessMethod() { @Override public MaterializeSchema readSchema() throws SQLException { - return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + // Materialize's information_schema is eventually consistent: tables and columns + // may not be visible immediately after creation. Retry until the snapshot is + // consistent. + readSchemaCallCount++; + for (int tries = 0; tries < 30; tries++) { + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() + .anyMatch(t -> t.getColumns().isEmpty()); + boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; + boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); + if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; } public void addFunctionAndType(String functionName, Character functionType) { From c7e8b04ebd169bc914e590b4407417138159799a Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 21 Apr 2026 09:15:55 +0800 Subject: [PATCH 782/856] Refactor: extract common INSERT generation logic into AbstractInsertGenerator Move shared boilerplate (INSERT INTO, column list, ExpectedErrors field) from 12 concrete InsertGenerator subclasses into AbstractInsertGenerator, reducing ~79 lines of duplicated code. Co-Authored-By: Claude Opus 4.6 --- .../gen/ClickHouseInsertGenerator.java | 10 +--------- .../common/gen/AbstractInsertGenerator.java | 20 ++++++++++++++++++- .../databend/gen/DatabendInsertGenerator.java | 12 +---------- .../gen/DataFusionInsertGenerator.java | 15 +------------- .../doris/gen/DorisInsertGenerator.java | 11 +--------- .../duckdb/gen/DuckDBInsertGenerator.java | 11 +--------- src/sqlancer/h2/H2InsertGenerator.java | 6 +----- .../hive/gen/HiveInsertGenerator.java | 2 -- .../hsqldb/gen/HSQLDBInsertGenerator.java | 12 +---------- .../presto/gen/PrestoInsertGenerator.java | 11 +--------- .../questdb/gen/QuestDBInsertGenerator.java | 13 +----------- .../spark/gen/SparkInsertGenerator.java | 2 -- .../ycql/gen/YCQLInsertGenerator.java | 12 +---------- 13 files changed, 29 insertions(+), 108 deletions(-) diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 9286185a7..593dec7c6 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -11,13 +11,11 @@ import sqlancer.clickhouse.ClickHouseSchema.ClickHouseTable; import sqlancer.clickhouse.ClickHouseToStringVisitor; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; public class ClickHouseInsertGenerator extends AbstractInsertGenerator { private final ClickHouseGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final ClickHouseExpressionGenerator gen; public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { @@ -42,13 +40,7 @@ private SQLQueryAdapter get() { columns = table.getRandomNonEmptyColumnSubset().stream().filter(c -> !c.isAlias() && !c.isMaterialized()) .collect(Collectors.toList()); } - sb.append("INSERT INTO "); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/common/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java index 1a0b2a997..7426656b8 100644 --- a/src/sqlancer/common/gen/AbstractInsertGenerator.java +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -1,12 +1,30 @@ package sqlancer.common.gen; import java.util.List; +import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractTableColumn; -public abstract class AbstractInsertGenerator { +public abstract class AbstractInsertGenerator> { protected StringBuilder sb = new StringBuilder(); + protected ExpectedErrors errors = new ExpectedErrors(); + + protected void appendColumnList(List columns) { + sb.append("("); + sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); + sb.append(")"); + } + + protected void buildInsertInto(String tableName, List columns) { + sb.append("INSERT INTO "); + sb.append(tableName); + appendColumnList(columns); + sb.append(" VALUES "); + insertColumns(columns); + } protected void insertColumns(List columns) { for (int nrRows = 0; nrRows < Randomly.smallNumber() + 1; nrRows++) { diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 340e494ef..6a3a4c51f 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -1,12 +1,9 @@ package sqlancer.databend.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendColumn; @@ -16,7 +13,6 @@ public class DatabendInsertGenerator extends AbstractInsertGenerator { private final DatabendGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DatabendInsertGenerator(DatabendGlobalState globalState) { this.globalState = globalState; @@ -27,15 +23,9 @@ public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); DatabendErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java index 1ee00dd50..43a340731 100644 --- a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.datafusion.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.IgnoreMeException; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.datafusion.DataFusionProvider.DataFusionGlobalState; import sqlancer.datafusion.DataFusionSchema.DataFusionColumn; @@ -15,7 +13,6 @@ public class DataFusionInsertGenerator extends AbstractInsertGenerator { private final DataFusionGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DataFusionInsertGenerator(DataFusionGlobalState globalState) { this.globalState = globalState; @@ -26,21 +23,11 @@ public static SQLQueryAdapter getQuery(DataFusionGlobalState globalState, DataFu } private SQLQueryAdapter generate(DataFusionTable targetTable) { - // `sb` is a global `StringBuilder` for current insert query - sb.append("INSERT INTO "); - if (targetTable.getColumns().isEmpty()) { throw new IgnoreMeException(); } List columns = targetTable.getRandomNonEmptyColumnSubset(); - - sb.append(targetTable.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); // will finally call `insertValue()` to generate random value - + buildInsertInto(targetTable.getName(), columns); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java index 50dc5cdec..9407ebf43 100644 --- a/src/sqlancer/doris/gen/DorisInsertGenerator.java +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.doris.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; @@ -16,7 +14,6 @@ public class DorisInsertGenerator extends AbstractInsertGenerator { private final DorisGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DorisInsertGenerator(DorisGlobalState globalState) { this.globalState = globalState; @@ -27,15 +24,9 @@ public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyInsertColumns(); - sb.append(table.getName()); - sb.append(" ("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); DorisErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 6793d2b51..65a9c222f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -1,11 +1,9 @@ package sqlancer.duckdb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -16,7 +14,6 @@ public class DuckDBInsertGenerator extends AbstractInsertGenerator { private final DuckDBGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public DuckDBInsertGenerator(DuckDBGlobalState globalState) { this.globalState = globalState; @@ -27,15 +24,9 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); DuckDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index c4e559b0e..be715315d 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -5,7 +5,6 @@ import sqlancer.Randomly; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Column; @@ -14,7 +13,6 @@ public class H2InsertGenerator extends AbstractInsertGenerator { private final H2GlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final H2ExpressionGenerator gen; public H2InsertGenerator(H2GlobalState globalState) { @@ -39,9 +37,7 @@ private SQLQueryAdapter generate() { H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); + appendColumnList(columns); if (mergeInto && Randomly.getBoolean()) { sb.append(" KEY("); sb.append(table.getRandomNonEmptyColumnSubset().stream().map(c -> c.getName()) diff --git a/src/sqlancer/hive/gen/HiveInsertGenerator.java b/src/sqlancer/hive/gen/HiveInsertGenerator.java index 963fafbce..cd0e11df7 100644 --- a/src/sqlancer/hive/gen/HiveInsertGenerator.java +++ b/src/sqlancer/hive/gen/HiveInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hive.HiveErrors; import sqlancer.hive.HiveGlobalState; @@ -14,7 +13,6 @@ public class HiveInsertGenerator extends AbstractInsertGenerator { private final HiveGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final HiveExpressionGenerator gen; public HiveInsertGenerator(HiveGlobalState globalState) { diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index 1cc132190..16020c508 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -1,10 +1,8 @@ package sqlancer.hsqldb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; @@ -14,7 +12,6 @@ public class HSQLDBInsertGenerator extends AbstractInsertGenerator { private final HSQLDBProvider.HSQLDBGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public HSQLDBInsertGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { this.globalState = globalState; @@ -25,16 +22,9 @@ public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalSt } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); - // HSQLDBErrors.addInsertErrors(errors); + buildInsertInto(table.getName(), columns); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index 072a22ae0..e9fa06f7b 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -1,10 +1,8 @@ package sqlancer.presto.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; @@ -26,16 +24,9 @@ public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); - ExpectedErrors errors = new ExpectedErrors(); + buildInsertInto(table.getName(), columns); PrestoErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors, false, false); } diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java index 754a0d2e5..e3a4dc35d 100644 --- a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -1,12 +1,9 @@ package sqlancer.questdb.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; import sqlancer.questdb.QuestDBErrors; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; @@ -17,22 +14,14 @@ public class QuestDBInsertGenerator extends AbstractInsertGenerator columns = table.getRandomNonEmptyColumnSubset(); - sb.append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(table.getName(), columns); QuestDBErrors.addInsertErrors(errors); return new SQLQueryAdapter(sb.toString(), errors); } diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java index b1755a848..c1404f509 100644 --- a/src/sqlancer/spark/gen/SparkInsertGenerator.java +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -3,7 +3,6 @@ import java.util.List; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.spark.SparkErrors; import sqlancer.spark.SparkGlobalState; @@ -14,7 +13,6 @@ public class SparkInsertGenerator extends AbstractInsertGenerator { private final SparkGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); private final SparkExpressionGenerator gen; public SparkInsertGenerator(SparkGlobalState globalState) { diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java index a1159d310..d53c70d29 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -1,12 +1,9 @@ package sqlancer.yugabyte.ycql.gen; import java.util.List; -import java.util.stream.Collectors; import sqlancer.common.gen.AbstractInsertGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ycql.YCQLErrors; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; @@ -16,7 +13,6 @@ public class YCQLInsertGenerator extends AbstractInsertGenerator { private final YCQLGlobalState globalState; - private final ExpectedErrors errors = new ExpectedErrors(); public YCQLInsertGenerator(YCQLGlobalState globalState) { this.globalState = globalState; @@ -27,15 +23,9 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { } private SQLQueryAdapter generate() { - sb.append("INSERT INTO "); YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getColumns(); - sb.append(globalState.getDatabaseName()).append(".").append(table.getName()); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES "); - insertColumns(columns); + buildInsertInto(globalState.getDatabaseName() + "." + table.getName(), columns); errors.add("Invalid Arguments"); errors.add("Null Argument for Primary Key"); From 38d14b423a31a46adb113c938fbce777854ac968 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Tue, 21 Apr 2026 10:52:45 +0800 Subject: [PATCH 783/856] Unify INSERT/UPDATE/DELETE generators under AbstractGenerator hierarchy Introduce AbstractDeleteGenerator extending AbstractGenerator, and make AbstractInsertGenerator and AbstractUpdateGenerator also extend it, eliminating duplicate sb/errors field declarations across all generator classes. Convert all 15 DELETE generators (CockroachDB, Databend, Doris, DuckDB, H2, MariaDB, Materialize, MySQL, OceanBase, Postgres, Presto, SQLite3, TiDB, YCQL, YSQL) from static-utility classes to the instance-based buildStatement() pattern. Add canonicalizeString field to AbstractGenerator (defaulting to true) so generators like Presto/Hive/Spark can opt out of semicolon canonicalization. Co-Authored-By: Claude Sonnet 4.6 --- .../gen/ClickHouseInsertGenerator.java | 9 +++--- .../gen/CockroachDBDeleteGenerator.java | 17 ++++++---- .../gen/CockroachDBIndexGenerator.java | 2 +- .../gen/CockroachDBTableGenerator.java | 2 +- .../gen/CockroachDBUpdateGenerator.java | 6 ++-- .../common/gen/AbstractDeleteGenerator.java | 5 +++ .../common/gen/AbstractGenerator.java | 5 +-- .../common/gen/AbstractInsertGenerator.java | 6 +--- .../common/gen/AbstractUpdateGenerator.java | 6 +--- .../databend/gen/DatabendDeleteGenerator.java | 18 +++++++---- .../databend/gen/DatabendInsertGenerator.java | 6 ++-- .../gen/DataFusionInsertGenerator.java | 10 +++--- .../doris/gen/DorisDeleteGenerator.java | 18 +++++++---- .../doris/gen/DorisInsertGenerator.java | 8 ++--- .../doris/gen/DorisUpdateGenerator.java | 6 ++-- .../duckdb/gen/DuckDBDeleteGenerator.java | 18 +++++++---- .../duckdb/gen/DuckDBInsertGenerator.java | 6 ++-- .../duckdb/gen/DuckDBUpdateGenerator.java | 6 ++-- src/sqlancer/h2/H2DeleteGenerator.java | 18 +++++++---- src/sqlancer/h2/H2InsertGenerator.java | 6 ++-- src/sqlancer/h2/H2UpdateGenerator.java | 6 ++-- .../hive/gen/HiveInsertGenerator.java | 7 ++-- .../hsqldb/gen/HSQLDBInsertGenerator.java | 6 ++-- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 6 ++-- .../mariadb/gen/MariaDBDeleteGenerator.java | 25 +++++++++------ .../gen/MaterializeDeleteGenerator.java | 18 +++++++---- .../gen/MaterializeUpdateGenerator.java | 8 ++--- .../mysql/gen/MySQLDeleteGenerator.java | 12 +++---- .../mysql/gen/MySQLUpdateGenerator.java | 10 +++--- .../gen/OceanBaseDeleteGenerator.java | 12 +++---- .../gen/OceanBaseUpdateGenerator.java | 7 ++-- .../postgres/gen/PostgresDeleteGenerator.java | 18 +++++++---- .../postgres/gen/PostgresUpdateGenerator.java | 8 ++--- .../presto/gen/PrestoDeleteGenerator.java | 19 +++++++---- .../presto/gen/PrestoInsertGenerator.java | 7 ++-- .../presto/gen/PrestoUpdateGenerator.java | 7 ++-- .../questdb/gen/QuestDBInsertGenerator.java | 12 +++---- .../spark/gen/SparkInsertGenerator.java | 7 ++-- .../gen/dml/SQLite3DeleteGenerator.java | 32 ++++++++++++------- .../gen/dml/SQLite3UpdateGenerator.java | 15 +++++---- .../tidb/gen/TiDBDeleteGenerator.java | 23 +++++++------ .../tidb/gen/TiDBUpdateGenerator.java | 10 +++--- .../ycql/gen/YCQLDeleteGenerator.java | 19 +++++++---- .../ycql/gen/YCQLInsertGenerator.java | 6 ++-- .../ycql/gen/YCQLUpdateGenerator.java | 6 ++-- .../ysql/gen/YSQLDeleteGenerator.java | 18 +++++++---- .../ysql/gen/YSQLUpdateGenerator.java | 8 ++--- 47 files changed, 291 insertions(+), 219 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractDeleteGenerator.java diff --git a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java index 593dec7c6..3951f4b55 100644 --- a/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java +++ b/src/sqlancer/clickhouse/gen/ClickHouseInsertGenerator.java @@ -1,6 +1,5 @@ package sqlancer.clickhouse.gen; -import java.sql.SQLException; import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -29,11 +28,12 @@ public ClickHouseInsertGenerator(ClickHouseGlobalState globalState) { ClickHouseErrors.addExpectedExpressionErrors(errors); } - public static SQLQueryAdapter getQuery(ClickHouseGlobalState globalState) throws SQLException { - return new ClickHouseInsertGenerator(globalState).get(); + public static SQLQueryAdapter getQuery(ClickHouseGlobalState globalState) { + return new ClickHouseInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter get() { + @Override + public void buildStatement() { ClickHouseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = Collections.emptyList(); while (columns.isEmpty()) { @@ -41,7 +41,6 @@ private SQLQueryAdapter get() { .collect(Collectors.toList()); } buildInsertInto(table.getName(), columns); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index 23fbae389..f0048d278 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -6,17 +6,23 @@ import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.CockroachDBVisitor; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; -public final class CockroachDBDeleteGenerator { +public final class CockroachDBDeleteGenerator extends AbstractDeleteGenerator { - private CockroachDBDeleteGenerator() { + private final CockroachDBGlobalState globalState; + + private CockroachDBDeleteGenerator(CockroachDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + return new CockroachDBDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append("DELETE FROM "); sb.append(table.getName()); @@ -30,7 +36,6 @@ public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { } errors.add("foreign key violation"); CockroachDBErrors.addTransactionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java index 4db3588ac..da6d3c609 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBIndexGenerator.java @@ -21,7 +21,7 @@ public static SQLQueryAdapter create(CockroachDBGlobalState s) { if (s.getSchema().getIndexCount() >= s.getDbmsSpecificOptions().maxNumIndexes) { throw new IgnoreMeException(); } - return new CockroachDBIndexGenerator(s).getQuery(); + return new CockroachDBIndexGenerator(s).getStatement(); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java index c8bfa3c6a..8678aff4d 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBTableGenerator.java @@ -32,7 +32,7 @@ public static SQLQueryAdapter generate(CockroachDBGlobalState globalState) { if (globalState.getSchema().getDatabaseTables().size() > globalState.getDbmsSpecificOptions().maxNumTables) { throw new IgnoreMeException(); } - return new CockroachDBTableGenerator(globalState).getQuery(); + return new CockroachDBTableGenerator(globalState).getStatement(); } @Override diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index 8dcd605d7..b367f5c59 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -22,10 +22,11 @@ private CockroachDBUpdateGenerator(CockroachDBGlobalState globalState) { } public static SQLQueryAdapter gen(CockroachDBGlobalState globalState) { - return new CockroachDBUpdateGenerator(globalState).generate(); + return new CockroachDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new CockroachDBExpressionGenerator(globalState).setColumns(columns); @@ -51,7 +52,6 @@ private SQLQueryAdapter generate() { errors.add("cannot write directly to computed column"); CockroachDBErrors.addExpressionErrors(errors); CockroachDBErrors.addTransactionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java new file mode 100644 index 000000000..47df974e6 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -0,0 +1,5 @@ +package sqlancer.common.gen; + +public abstract class AbstractDeleteGenerator extends AbstractGenerator { + +} diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java index c478610c4..5d13fc746 100644 --- a/src/sqlancer/common/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -8,10 +8,11 @@ public abstract class AbstractGenerator { protected final ExpectedErrors errors = new ExpectedErrors(); protected final StringBuilder sb = new StringBuilder(); protected boolean canAffectSchema; + protected boolean canonicalizeString = true; - public SQLQueryAdapter getQuery() { + public SQLQueryAdapter getStatement() { buildStatement(); - return new SQLQueryAdapter(sb.toString(), errors, canAffectSchema); + return new SQLQueryAdapter(sb.toString(), errors, canAffectSchema, canonicalizeString); } public abstract void buildStatement(); diff --git a/src/sqlancer/common/gen/AbstractInsertGenerator.java b/src/sqlancer/common/gen/AbstractInsertGenerator.java index 7426656b8..1a1b36b67 100644 --- a/src/sqlancer/common/gen/AbstractInsertGenerator.java +++ b/src/sqlancer/common/gen/AbstractInsertGenerator.java @@ -4,13 +4,9 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; -public abstract class AbstractInsertGenerator> { - - protected StringBuilder sb = new StringBuilder(); - protected ExpectedErrors errors = new ExpectedErrors(); +public abstract class AbstractInsertGenerator> extends AbstractGenerator { protected void appendColumnList(List columns) { sb.append("("); diff --git a/src/sqlancer/common/gen/AbstractUpdateGenerator.java b/src/sqlancer/common/gen/AbstractUpdateGenerator.java index f130c15a5..52b716bff 100644 --- a/src/sqlancer/common/gen/AbstractUpdateGenerator.java +++ b/src/sqlancer/common/gen/AbstractUpdateGenerator.java @@ -2,13 +2,9 @@ import java.util.List; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.schema.AbstractTableColumn; -public abstract class AbstractUpdateGenerator> { - - protected final ExpectedErrors errors = new ExpectedErrors(); - protected StringBuilder sb = new StringBuilder(); +public abstract class AbstractUpdateGenerator> extends AbstractGenerator { protected void updateColumns(List columns) { for (int nrColumn = 0; nrColumn < columns.size(); nrColumn++) { diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index 256030409..5adabf175 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -1,21 +1,28 @@ package sqlancer.databend.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendDataType; import sqlancer.databend.DatabendToStringVisitor; -public final class DatabendDeleteGenerator { +public final class DatabendDeleteGenerator extends AbstractDeleteGenerator { - private DatabendDeleteGenerator() { + private final DatabendGlobalState globalState; + + private DatabendDeleteGenerator(DatabendGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(DatabendGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new DatabendDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); @@ -23,7 +30,6 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN))); DatabendErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/databend/gen/DatabendInsertGenerator.java b/src/sqlancer/databend/gen/DatabendInsertGenerator.java index 6a3a4c51f..13ee80e95 100644 --- a/src/sqlancer/databend/gen/DatabendInsertGenerator.java +++ b/src/sqlancer/databend/gen/DatabendInsertGenerator.java @@ -19,15 +19,15 @@ public DatabendInsertGenerator(DatabendGlobalState globalState) { } public static SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - return new DatabendInsertGenerator(globalState).generate(); + return new DatabendInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DatabendTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); DatabendErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java index 43a340731..36a178791 100644 --- a/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java +++ b/src/sqlancer/datafusion/gen/DataFusionInsertGenerator.java @@ -13,22 +13,24 @@ public class DataFusionInsertGenerator extends AbstractInsertGenerator { private final DataFusionGlobalState globalState; + private final DataFusionTable targetTable; - public DataFusionInsertGenerator(DataFusionGlobalState globalState) { + public DataFusionInsertGenerator(DataFusionGlobalState globalState, DataFusionTable targetTable) { this.globalState = globalState; + this.targetTable = targetTable; } public static SQLQueryAdapter getQuery(DataFusionGlobalState globalState, DataFusionTable targetTable) { - return new DataFusionInsertGenerator(globalState).generate(targetTable); + return new DataFusionInsertGenerator(globalState, targetTable).getStatement(); } - private SQLQueryAdapter generate(DataFusionTable targetTable) { + @Override + public void buildStatement() { if (targetTable.getColumns().isEmpty()) { throw new IgnoreMeException(); } List columns = targetTable.getRandomNonEmptyColumnSubset(); buildInsertInto(targetTable.getName(), columns); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/doris/gen/DorisDeleteGenerator.java b/src/sqlancer/doris/gen/DorisDeleteGenerator.java index 27f369aec..66deaea8f 100644 --- a/src/sqlancer/doris/gen/DorisDeleteGenerator.java +++ b/src/sqlancer/doris/gen/DorisDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.doris.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.doris.DorisErrors; import sqlancer.doris.DorisProvider.DorisGlobalState; @@ -9,14 +9,21 @@ import sqlancer.doris.DorisSchema.DorisTable; import sqlancer.doris.visitor.DorisToStringVisitor; -public final class DorisDeleteGenerator { +public final class DorisDeleteGenerator extends AbstractDeleteGenerator { - private DorisDeleteGenerator() { + private final DorisGlobalState globalState; + + private DorisDeleteGenerator(DorisGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(DorisGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new DorisDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -25,7 +32,6 @@ public static SQLQueryAdapter generate(DorisGlobalState globalState) { .setColumns(table.getColumns()).generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/doris/gen/DorisInsertGenerator.java b/src/sqlancer/doris/gen/DorisInsertGenerator.java index 9407ebf43..e05bc85fd 100644 --- a/src/sqlancer/doris/gen/DorisInsertGenerator.java +++ b/src/sqlancer/doris/gen/DorisInsertGenerator.java @@ -20,15 +20,15 @@ public DorisInsertGenerator(DorisGlobalState globalState) { } public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { - return new DorisInsertGenerator(globalState).generate(); + return new DorisInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - List columns = table.getRandomNonEmptyInsertColumns(); + List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); DorisErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java index 906173921..e0db2b7b4 100644 --- a/src/sqlancer/doris/gen/DorisUpdateGenerator.java +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -23,10 +23,11 @@ private DorisUpdateGenerator(DorisGlobalState globalState) { } public static SQLQueryAdapter getQuery(DorisGlobalState globalState) { - return new DorisUpdateGenerator(globalState).generate(); + return new DorisUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new DorisNewExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -37,7 +38,6 @@ private SQLQueryAdapter generate() { sb.append(" WHERE "); sb.append(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index 5fadce30a..42695a9f6 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -1,21 +1,28 @@ package sqlancer.duckdb.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; import sqlancer.duckdb.DuckDBToStringVisitor; -public final class DuckDBDeleteGenerator { +public final class DuckDBDeleteGenerator extends AbstractDeleteGenerator { - private DuckDBDeleteGenerator() { + private final DuckDBGlobalState globalState; + + private DuckDBDeleteGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new DuckDBDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -24,7 +31,6 @@ public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } DuckDBErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java index 65a9c222f..e8b122a51 100644 --- a/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBInsertGenerator.java @@ -20,15 +20,15 @@ public DuckDBInsertGenerator(DuckDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - return new DuckDBInsertGenerator(globalState).generate(); + return new DuckDBInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); buildInsertInto(table.getName(), columns); DuckDBErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java index b4ffd0140..8e2ddd047 100644 --- a/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBUpdateGenerator.java @@ -22,10 +22,11 @@ private DuckDBUpdateGenerator(DuckDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - return new DuckDBUpdateGenerator(globalState).generate(); + return new DuckDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubsetFilter(p -> !p.getName().equals("rowid")); gen = new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -34,7 +35,6 @@ private SQLQueryAdapter generate() { sb.append(" SET "); updateColumns(columns); DuckDBErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java index 58afdcb48..dd0fdff34 100644 --- a/src/sqlancer/h2/H2DeleteGenerator.java +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -1,19 +1,26 @@ package sqlancer.h2; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.h2.H2Provider.H2GlobalState; import sqlancer.h2.H2Schema.H2Table; -public final class H2DeleteGenerator { +public final class H2DeleteGenerator extends AbstractDeleteGenerator { - private H2DeleteGenerator() { + private final H2GlobalState globalState; + + private H2DeleteGenerator(H2GlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new H2DeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -27,7 +34,6 @@ public static SQLQueryAdapter getQuery(H2GlobalState globalState) { } H2Errors.addExpressionErrors(errors); H2Errors.addDeleteErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/h2/H2InsertGenerator.java b/src/sqlancer/h2/H2InsertGenerator.java index be715315d..3bd7552d8 100644 --- a/src/sqlancer/h2/H2InsertGenerator.java +++ b/src/sqlancer/h2/H2InsertGenerator.java @@ -21,10 +21,11 @@ public H2InsertGenerator(H2GlobalState globalState) { } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - return new H2InsertGenerator(globalState).generate(); + return new H2InsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { boolean mergeInto = false; // Randomly.getBooleanWithRatherLowProbability(); if (mergeInto) { sb.append("MERGE INTO "); @@ -48,7 +49,6 @@ private SQLQueryAdapter generate() { insertColumns(columns); H2Errors.addInsertErrors(errors); H2Errors.addExpressionErrors(errors); // generated columns - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index 158621409..fe63c7a4e 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -19,10 +19,11 @@ private H2UpdateGenerator(H2GlobalState globalState) { } public static SQLQueryAdapter getQuery(H2GlobalState globalState) { - return new H2UpdateGenerator(globalState).generate(); + return new H2UpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new H2ExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -37,7 +38,6 @@ private SQLQueryAdapter generate() { sb.append(H2ToStringVisitor.asString(gen.generateExpression())); } H2Errors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/hive/gen/HiveInsertGenerator.java b/src/sqlancer/hive/gen/HiveInsertGenerator.java index cd0e11df7..8c23e4456 100644 --- a/src/sqlancer/hive/gen/HiveInsertGenerator.java +++ b/src/sqlancer/hive/gen/HiveInsertGenerator.java @@ -18,10 +18,11 @@ public class HiveInsertGenerator extends AbstractInsertGenerator { public HiveInsertGenerator(HiveGlobalState globalState) { this.globalState = globalState; this.gen = new HiveExpressionGenerator(globalState); + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(HiveGlobalState globalState) { - return new HiveInsertGenerator(globalState).generate(); + return new HiveInsertGenerator(globalState).getStatement(); } @Override @@ -29,7 +30,8 @@ protected void insertValue(HiveColumn column) { sb.append(HiveToStringVisitor.asString(gen.generateConstant())); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { // Inserting values into tables from SQL. sb.append("INSERT INTO "); HiveTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); @@ -45,6 +47,5 @@ private SQLQueryAdapter generate() { insertColumns(columns); HiveErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java index 16020c508..00d99be38 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBInsertGenerator.java @@ -18,14 +18,14 @@ public HSQLDBInsertGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { - return new HSQLDBInsertGenerator(globalState).generate(); + return new HSQLDBInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index e639e21b3..54380214f 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -24,10 +24,11 @@ private HSQLDBUpdateGenerator(HSQLDBProvider.HSQLDBGlobalState globalState) { } public static SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState) { - return new HSQLDBUpdateGenerator(globalState).generate(); + return new HSQLDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { HSQLDBSchema.HSQLDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new HSQLDBExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -42,7 +43,6 @@ private SQLQueryAdapter generate() { errors.add("data type of expression is not boolean"); HSQLDBErrors.addExpressionErrors(errors); } - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index 6d85eb891..2992f569d 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -3,7 +3,7 @@ import java.util.Collections; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTables; import sqlancer.mariadb.MariaDBSchema; @@ -11,12 +11,22 @@ import sqlancer.mariadb.MariaDBSchema.MariaDBTable; import sqlancer.mariadb.ast.MariaDBVisitor; -public final class MariaDBDeleteGenerator { +public final class MariaDBDeleteGenerator extends AbstractDeleteGenerator { - private MariaDBDeleteGenerator() { + private final MariaDBSchema schema; + private final Randomly r; + + private MariaDBDeleteGenerator(MariaDBSchema schema, Randomly r) { + this.schema = schema; + this.r = r; } public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { + return new MariaDBDeleteGenerator(schema, r).getStatement(); + } + + @Override + public void buildStatement() { MariaDBTable table = schema.getRandomTable(); MariaDBExpressionGenerator expressionGenerator = new MariaDBExpressionGenerator(r); @@ -25,15 +35,13 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { Collections.singletonList(table)); expressionGenerator.setTablesAndColumns(tablesAndColumns); - ExpectedErrors errors = new ExpectedErrors(); - errors.add("foreign key constraint fails"); errors.add("cannot delete or update a parent row"); errors.add("Data truncated"); errors.add("Division by 0"); errors.add("Incorrect value"); - StringBuilder sb = new StringBuilder("DELETE"); + sb.append("DELETE"); if (Randomly.getBooleanWithRatherLowProbability()) { sb.append(" LOW_PRIORITY"); @@ -81,13 +89,10 @@ public static SQLQueryAdapter delete(MariaDBSchema schema, Randomly r) { } } - String query = sb.toString(); - if (query.contains("RLIKE") || query.contains("REGEXP")) { + if (sb.toString().contains("RLIKE") || sb.toString().contains("REGEXP")) { errors.add("Regex error"); errors.add("quantifier does not follow a repeatable item"); errors.add("Got error"); } - - return new SQLQueryAdapter(query, errors); } } diff --git a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java index ded3e53f4..2aceddcc9 100644 --- a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java @@ -1,25 +1,32 @@ package sqlancer.materialize.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.materialize.MaterializeGlobalState; import sqlancer.materialize.MaterializeSchema.MaterializeDataType; import sqlancer.materialize.MaterializeSchema.MaterializeTable; import sqlancer.materialize.MaterializeVisitor; -public final class MaterializeDeleteGenerator { +public final class MaterializeDeleteGenerator extends AbstractDeleteGenerator { - private MaterializeDeleteGenerator() { + private final MaterializeGlobalState globalState; + + private MaterializeDeleteGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter create(MaterializeGlobalState globalState) { + return new MaterializeDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { MaterializeTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append("DELETE FROM"); sb.append(" "); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -32,7 +39,6 @@ public static SQLQueryAdapter create(MaterializeGlobalState globalState) { errors.add("does not support casting"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java index 1b7e69208..7d338c027 100644 --- a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java @@ -20,6 +20,7 @@ public final class MaterializeUpdateGenerator extends AbstractUpdateGenerator t.isInsertable()); List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); @@ -55,8 +57,6 @@ private SQLQueryAdapter generate() { randomTable.getColumns(), MaterializeDataType.BOOLEAN); sb.append(MaterializeVisitor.asString(where)); } - - return new SQLQueryAdapter(sb.toString(), errors, true); } @Override diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index f12c23a72..048ef335f 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -3,16 +3,15 @@ import java.util.Arrays; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; -public class MySQLDeleteGenerator { +public class MySQLDeleteGenerator extends AbstractDeleteGenerator { - private final StringBuilder sb = new StringBuilder(); private final MySQLGlobalState globalState; public MySQLDeleteGenerator(MySQLGlobalState globalState) { @@ -20,13 +19,13 @@ public MySQLDeleteGenerator(MySQLGlobalState globalState) { } public static SQLQueryAdapter delete(MySQLGlobalState globalState) { - return new MySQLDeleteGenerator(globalState).generate(); + return new MySQLDeleteGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { MySQLTable randomTable = globalState.getSchema().getRandomTable(); MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()); - ExpectedErrors errors = new ExpectedErrors(); sb.append("DELETE"); if (Randomly.getBoolean()) { sb.append(" LOW_PRIORITY"); @@ -51,7 +50,6 @@ private SQLQueryAdapter generate() { */, "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index")); // TODO: support ORDER BY - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java index 55ba3dd45..1aca84580 100644 --- a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java @@ -1,6 +1,5 @@ package sqlancer.mysql.gen; -import java.sql.SQLException; import java.util.List; import sqlancer.Randomly; @@ -21,11 +20,12 @@ public MySQLUpdateGenerator(MySQLGlobalState globalState) { this.globalState = globalState; } - public static SQLQueryAdapter create(MySQLGlobalState globalState) throws SQLException { - return new MySQLUpdateGenerator(globalState).generate(); + public static SQLQueryAdapter create(MySQLGlobalState globalState) { + return new MySQLUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() throws SQLException { + @Override + public void buildStatement() { MySQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new MySQLExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -40,8 +40,6 @@ private SQLQueryAdapter generate() throws SQLException { } MySQLErrors.addInsertUpdateErrors(errors); errors.add("doesn't have this option"); - - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index ea1cb36e8..c46ace304 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -3,16 +3,15 @@ import java.util.Arrays; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.oceanbase.OceanBaseErrors; import sqlancer.oceanbase.OceanBaseGlobalState; import sqlancer.oceanbase.OceanBaseSchema.OceanBaseTable; import sqlancer.oceanbase.OceanBaseVisitor; -public class OceanBaseDeleteGenerator { +public class OceanBaseDeleteGenerator extends AbstractDeleteGenerator { - private final StringBuilder sb = new StringBuilder(); private final OceanBaseGlobalState globalState; private final Randomly r; @@ -22,14 +21,14 @@ public OceanBaseDeleteGenerator(OceanBaseGlobalState globalState) { } public static SQLQueryAdapter delete(OceanBaseGlobalState globalState) { - return new OceanBaseDeleteGenerator(globalState).generate(); + return new OceanBaseDeleteGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { OceanBaseTable randomTable = globalState.getSchema().getRandomTable(); OceanBaseExpressionGenerator gen = new OceanBaseExpressionGenerator(globalState) .setColumns(randomTable.getColumns()); - ExpectedErrors errors = new ExpectedErrors(); sb.append("DELETE"); if (Randomly.getBoolean()) { sb.append(" /*+parallel(" + r.getLong(0, 10) + ") enable_parallel_dml*/ "); @@ -45,7 +44,6 @@ private SQLQueryAdapter generate() { "Truncated incorrect INTEGER value", "Truncated incorrect DECIMAL value", "Data truncated for functional index", "Incorrect value", "Out of range value for column", "Data truncation:")); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index 950317bd2..44fdf39fe 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -23,10 +23,11 @@ public OceanBaseUpdateGenerator(OceanBaseGlobalState globalState) { } public static SQLQueryAdapter update(OceanBaseGlobalState globalState) { - return new OceanBaseUpdateGenerator(globalState).generate(); + return new OceanBaseUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { OceanBaseSchema.OceanBaseTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new OceanBaseExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -45,8 +46,6 @@ private SQLQueryAdapter generate() { } errors.add("Duplicated primary key"); OceanBaseErrors.addInsertErrors(errors); - - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index f827331c0..452f5cdf4 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -1,25 +1,32 @@ package sqlancer.postgres.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresDataType; import sqlancer.postgres.PostgresSchema.PostgresTable; import sqlancer.postgres.PostgresVisitor; -public final class PostgresDeleteGenerator { +public final class PostgresDeleteGenerator extends AbstractDeleteGenerator { - private PostgresDeleteGenerator() { + private final PostgresGlobalState globalState; + + private PostgresDeleteGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter create(PostgresGlobalState globalState) { + return new PostgresDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { PostgresTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append("DELETE FROM"); if (Randomly.getBoolean()) { sb.append(" ONLY"); } @@ -40,7 +47,6 @@ public static SQLQueryAdapter create(PostgresGlobalState globalState) { errors.add("cannot cast"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 7ce7fe882..1effad582 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -20,6 +20,7 @@ public final class PostgresUpdateGenerator extends AbstractUpdateGenerator t.isInsertable()); List columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); @@ -55,8 +57,6 @@ private SQLQueryAdapter generate() { randomTable.getColumns(), PostgresDataType.BOOLEAN); sb.append(PostgresVisitor.asString(where)); } - - return new SQLQueryAdapter(sb.toString(), errors, true); } @Override diff --git a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java index 9f869c241..59b7b4174 100644 --- a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java +++ b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.presto.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoErrors; import sqlancer.presto.PrestoGlobalState; @@ -9,14 +9,22 @@ import sqlancer.presto.PrestoSchema.PrestoTable; import sqlancer.presto.PrestoToStringVisitor; -public final class PrestoDeleteGenerator { +public final class PrestoDeleteGenerator extends AbstractDeleteGenerator { - private PrestoDeleteGenerator() { + private final PrestoGlobalState globalState; + + private PrestoDeleteGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canonicalizeString = false; } public static SQLQueryAdapter generate(PrestoGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new PrestoDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + sb.append("DELETE FROM "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); if (Randomly.getBoolean()) { @@ -26,7 +34,6 @@ public static SQLQueryAdapter generate(PrestoGlobalState globalState) { .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()))); } PrestoErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } } diff --git a/src/sqlancer/presto/gen/PrestoInsertGenerator.java b/src/sqlancer/presto/gen/PrestoInsertGenerator.java index e9fa06f7b..15d5f3543 100644 --- a/src/sqlancer/presto/gen/PrestoInsertGenerator.java +++ b/src/sqlancer/presto/gen/PrestoInsertGenerator.java @@ -17,18 +17,19 @@ public class PrestoInsertGenerator extends AbstractInsertGenerator public PrestoInsertGenerator(PrestoGlobalState globalState) { this.globalState = globalState; + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { - return new PrestoInsertGenerator(globalState).generate(); + return new PrestoInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); PrestoErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } @Override diff --git a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java index a8afcc578..3c197f0b9 100644 --- a/src/sqlancer/presto/gen/PrestoUpdateGenerator.java +++ b/src/sqlancer/presto/gen/PrestoUpdateGenerator.java @@ -19,13 +19,15 @@ public final class PrestoUpdateGenerator extends AbstractUpdateGenerator !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -34,7 +36,6 @@ private SQLQueryAdapter generate() { sb.append(" SET "); updateColumns(columns); PrestoErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } @Override diff --git a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java index e3a4dc35d..41ba72406 100644 --- a/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBInsertGenerator.java @@ -18,16 +18,16 @@ public QuestDBInsertGenerator(QuestDBGlobalState globalState) { this.globalState = globalState; } - private SQLQueryAdapter generate() { + public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { + return new QuestDBInsertGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { QuestDBTable table = globalState.getSchema().getRandomTable(); List columns = table.getRandomNonEmptyColumnSubset(); buildInsertInto(table.getName(), columns); QuestDBErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); - } - - public static SQLQueryAdapter getQuery(QuestDBGlobalState globalState) { - return new QuestDBInsertGenerator(globalState).generate(); } @Override diff --git a/src/sqlancer/spark/gen/SparkInsertGenerator.java b/src/sqlancer/spark/gen/SparkInsertGenerator.java index c1404f509..c43584315 100644 --- a/src/sqlancer/spark/gen/SparkInsertGenerator.java +++ b/src/sqlancer/spark/gen/SparkInsertGenerator.java @@ -18,10 +18,11 @@ public class SparkInsertGenerator extends AbstractInsertGenerator { public SparkInsertGenerator(SparkGlobalState globalState) { this.globalState = globalState; this.gen = new SparkExpressionGenerator(globalState); + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(SparkGlobalState globalState) { - return new SparkInsertGenerator(globalState).generate(); + return new SparkInsertGenerator(globalState).getStatement(); } @Override @@ -29,7 +30,8 @@ protected void insertValue(SparkColumn column) { sb.append(SparkToStringVisitor.asString(gen.generateConstant())); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { sb.append("INSERT INTO "); SparkTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); @@ -40,6 +42,5 @@ private SQLQueryAdapter generate() { insertColumns(columns); SparkErrors.addInsertErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, false, false); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 4c86417d0..51b23eaa4 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -3,7 +3,7 @@ import java.util.Arrays; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.sqlite3.SQLite3Errors; import sqlancer.sqlite3.SQLite3GlobalState; @@ -11,26 +11,35 @@ import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public final class SQLite3DeleteGenerator { +public final class SQLite3DeleteGenerator extends AbstractDeleteGenerator { - private SQLite3DeleteGenerator() { + private final SQLite3GlobalState globalState; + private final SQLite3Table table; + + private SQLite3DeleteGenerator(SQLite3GlobalState globalState, SQLite3Table table) { + this.globalState = globalState; + this.table = table; + this.canAffectSchema = true; } public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState) { - SQLite3Table tableName = globalState.getSchema().getRandomTable(t -> !t.isView() && !t.isReadOnly()); - return deleteContent(globalState, tableName); + SQLite3Table table = globalState.getSchema().getRandomTable(t -> !t.isView() && !t.isReadOnly()); + return deleteContent(globalState, table); + } + + public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLite3Table table) { + return new SQLite3DeleteGenerator(globalState, table).getStatement(); } - public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLite3Table tableName) { - StringBuilder sb = new StringBuilder(); + @Override + public void buildStatement() { sb.append("DELETE FROM "); - sb.append(tableName.getName()); + sb.append(table.getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); - sb.append(SQLite3Visitor.asString(new SQLite3ExpressionGenerator(globalState) - .setColumns(tableName.getColumns()).generateExpression())); + sb.append(SQLite3Visitor.asString( + new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } - ExpectedErrors errors = new ExpectedErrors(); SQLite3Errors.addExpectedExpressionErrors(errors); errors.addAll(Arrays.asList("[SQLITE_ERROR] SQL error or missing database (foreign key mismatch", "[SQLITE_CONSTRAINT] Abort due to constraint violation ", @@ -40,7 +49,6 @@ public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLi "cannot INSERT into generated column", "A table in the database is locked", "load_extension() prohibited in triggers and views", "The database file is locked")); SQLite3Errors.addDeleteErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 5a17ad339..9a2be3150 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -18,10 +18,13 @@ public class SQLite3UpdateGenerator extends AbstractUpdateGenerator columnsToUpdate = Randomly.nonEmptySubsetPotentialDuplicates(table.getColumns()); sb.append("UPDATE "); if (Randomly.getBoolean()) { @@ -98,8 +101,6 @@ private SQLQueryAdapter generate(SQLite3Table table) { SQLite3Errors.addInsertNowErrors(errors); SQLite3Errors.addExpectedExpressionErrors(errors); SQLite3Errors.addDeleteErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true /* column could have an ON UPDATE clause */); - } @Override diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index c3986f8d9..cbe5fc721 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -1,10 +1,9 @@ package sqlancer.tidb.gen; -import java.sql.SQLException; import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBErrors; import sqlancer.tidb.TiDBExpressionGenerator; @@ -12,16 +11,24 @@ import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.visitor.TiDBVisitor; -public final class TiDBDeleteGenerator { +public final class TiDBDeleteGenerator extends AbstractDeleteGenerator { - private TiDBDeleteGenerator() { + private final TiDBGlobalState globalState; + + private TiDBDeleteGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + } + + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + return new TiDBDeleteGenerator(globalState).getStatement(); } - public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - ExpectedErrors errors = ExpectedErrors.newErrors().with(TiDBErrors.getExpressionErrors()).build(); + @Override + public void buildStatement() { + errors.addAll(TiDBErrors.getExpressionErrors()); TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); TiDBExpressionGenerator gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); - StringBuilder sb = new StringBuilder("DELETE "); + sb.append("DELETE "); if (Randomly.getBooleanWithSmallProbability()) { sb.append("LOW_PRIORITY "); } @@ -55,8 +62,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx errors.add("is not valid for CHARACTER SET"); errors.add("Division by 0"); errors.add("error parsing regexp"); - return new SQLQueryAdapter(sb.toString(), errors); - } } diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index 241ee3321..af6430b48 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -1,6 +1,5 @@ package sqlancer.tidb.gen; -import java.sql.SQLException; import java.util.List; import sqlancer.Randomly; @@ -22,11 +21,12 @@ private TiDBUpdateGenerator(TiDBGlobalState globalState) { this.globalState = globalState; } - public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { - return new TiDBUpdateGenerator(globalState).generate(); + public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) { + return new TiDBUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() throws SQLException { + @Override + public void buildStatement() { TiDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new TiDBExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -40,8 +40,6 @@ private SQLQueryAdapter generate() throws SQLException { sb.append(TiDBVisitor.asString(gen.generateExpression())); } TiDBErrors.addInsertErrors(errors); - - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java index 108cd1be9..6ebe0db5f 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java @@ -1,31 +1,36 @@ package sqlancer.yugabyte.ycql.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLErrors; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLTable; import sqlancer.yugabyte.ycql.YCQLToStringVisitor; -public final class YCQLDeleteGenerator { +public final class YCQLDeleteGenerator extends AbstractDeleteGenerator { - private YCQLDeleteGenerator() { + private final YCQLGlobalState globalState; + + private YCQLDeleteGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(YCQLGlobalState globalState) { - StringBuilder sb = new StringBuilder("DELETE FROM "); - ExpectedErrors errors = new ExpectedErrors(); + return new YCQLDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); + sb.append("DELETE FROM "); sb.append(table.getName()); if (Randomly.getBoolean()) { sb.append(" WHERE "); sb.append(YCQLToStringVisitor.asString( new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } - YCQLErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java index d53c70d29..167f5d237 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLInsertGenerator.java @@ -19,10 +19,11 @@ public YCQLInsertGenerator(YCQLGlobalState globalState) { } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - return new YCQLInsertGenerator(globalState).generate(); + return new YCQLInsertGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getColumns(); buildInsertInto(globalState.getDatabaseName() + "." + table.getName(), columns); @@ -31,7 +32,6 @@ private SQLQueryAdapter generate() { errors.add("Null Argument for Primary Key"); YCQLErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java index a6c855cf3..eee49bee4 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLUpdateGenerator.java @@ -22,10 +22,11 @@ private YCQLUpdateGenerator(YCQLGlobalState globalState) { } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - return new YCQLUpdateGenerator(globalState).generate(); + return new YCQLUpdateGenerator(globalState).getStatement(); } - private SQLQueryAdapter generate() { + @Override + public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); List columns = table.getRandomNonEmptyColumnSubset(); gen = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()); @@ -41,7 +42,6 @@ private SQLQueryAdapter generate() { errors.add("Missing Argument for Primary Key"); YCQLErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } @Override diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java index e0128707b..6e35c6862 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -1,7 +1,7 @@ package sqlancer.yugabyte.ysql.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractDeleteGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ysql.YSQLErrors; import sqlancer.yugabyte.ysql.YSQLGlobalState; @@ -9,18 +9,25 @@ import sqlancer.yugabyte.ysql.YSQLSchema.YSQLTable; import sqlancer.yugabyte.ysql.YSQLVisitor; -public final class YSQLDeleteGenerator { +public final class YSQLDeleteGenerator extends AbstractDeleteGenerator { - private YSQLDeleteGenerator() { + private final YSQLGlobalState globalState; + + private YSQLDeleteGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter create(YSQLGlobalState globalState) { + return new YSQLDeleteGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { YSQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - ExpectedErrors errors = new ExpectedErrors(); errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - StringBuilder sb = new StringBuilder("DELETE FROM"); + sb.append("DELETE FROM"); if (Randomly.getBoolean()) { sb.append(" ONLY"); } @@ -41,7 +48,6 @@ public static SQLQueryAdapter create(YSQLGlobalState globalState) { errors.add("cannot cast"); errors.add("invalid input syntax for"); errors.add("division by zero"); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index bc7b00d79..6c5fd4144 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -21,6 +21,7 @@ public final class YSQLUpdateGenerator extends AbstractUpdateGenerator columns = randomTable.getRandomNonEmptyColumnSubset(); sb.append("UPDATE "); @@ -57,8 +59,6 @@ private SQLQueryAdapter generate() { YSQLDataType.BOOLEAN); sb.append(YSQLVisitor.asString(where)); } - - return new SQLQueryAdapter(sb.toString(), errors, true); } @Override From 5d2e48a932d0fa17f0fc32ab4bd088635ca426b7 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Apr 2026 00:35:56 +0800 Subject: [PATCH 784/856] Address review feedback: fix comment and add MaterializeBugs Remove incorrect claim about eventual consistency in readSchema comment. Add MaterializeBugs class with bugSchemaReadIncomplete flag to guard the retry logic, following the MySQLBugs pattern. Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/materialize/MaterializeBugs.java | 12 ++++++++++++ src/sqlancer/materialize/MaterializeGlobalState.java | 8 +++++--- 2 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 src/sqlancer/materialize/MaterializeBugs.java diff --git a/src/sqlancer/materialize/MaterializeBugs.java b/src/sqlancer/materialize/MaterializeBugs.java new file mode 100644 index 000000000..a7611a925 --- /dev/null +++ b/src/sqlancer/materialize/MaterializeBugs.java @@ -0,0 +1,12 @@ +package sqlancer.materialize; + +// do not make the fields final to avoid warnings +public final class MaterializeBugs { + + // Tables or columns may be missing when reading information_schema shortly after creation + public static boolean bugSchemaReadIncomplete = true; + + private MaterializeBugs() { + } + +} diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java index 8a66a7819..46977529b 100644 --- a/src/sqlancer/materialize/MaterializeGlobalState.java +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -268,10 +268,12 @@ public String getRandomTableAccessMethod() { @Override public MaterializeSchema readSchema() throws SQLException { - // Materialize's information_schema is eventually consistent: tables and columns - // may not be visible immediately after creation. Retry until the snapshot is - // consistent. + // Workaround for a suspected Materialize bug where tables or columns may be + // missing when reading the schema; retry until stable. readSchemaCallCount++; + if (!MaterializeBugs.bugSchemaReadIncomplete) { + return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + } for (int tries = 0; tries < 30; tries++) { MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() From 82612b1d0f310cc131661d2b92a8d8571453a2f9 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Apr 2026 22:16:13 +0800 Subject: [PATCH 785/856] Wrap readSchema workaround in bugSchemaReadIncomplete guard Co-Authored-By: Claude Opus 4.7 --- .../materialize/MaterializeGlobalState.java | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/sqlancer/materialize/MaterializeGlobalState.java b/src/sqlancer/materialize/MaterializeGlobalState.java index 46977529b..7ab963c03 100644 --- a/src/sqlancer/materialize/MaterializeGlobalState.java +++ b/src/sqlancer/materialize/MaterializeGlobalState.java @@ -268,32 +268,32 @@ public String getRandomTableAccessMethod() { @Override public MaterializeSchema readSchema() throws SQLException { - // Workaround for a suspected Materialize bug where tables or columns may be - // missing when reading the schema; retry until stable. - readSchemaCallCount++; - if (!MaterializeBugs.bugSchemaReadIncomplete) { - return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); - } - for (int tries = 0; tries < 30; tries++) { - MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); - boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() - .anyMatch(t -> t.getColumns().isEmpty()); - boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; - boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); - if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { - lastKnownTableCount = schema.getDatabaseTables().size(); - return schema; - } - try { - Thread.sleep(100); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; + if (MaterializeBugs.bugSchemaReadIncomplete) { + // Workaround for a suspected Materialize bug where tables or columns may be + // missing when reading the schema; retry until stable. + readSchemaCallCount++; + for (int tries = 0; tries < 30; tries++) { + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + boolean hasTableWithEmptyColumns = schema.getDatabaseTables().stream() + .anyMatch(t -> t.getColumns().isEmpty()); + boolean tableCountRegressed = schema.getDatabaseTables().size() < lastKnownTableCount; + boolean suspiciouslyEmpty = readSchemaCallCount > 1 && schema.getDatabaseTables().isEmpty(); + if (!hasTableWithEmptyColumns && !tableCountRegressed && !suspiciouslyEmpty) { + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; + } + try { + Thread.sleep(100); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } } + MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); + lastKnownTableCount = schema.getDatabaseTables().size(); + return schema; } - MaterializeSchema schema = MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); - lastKnownTableCount = schema.getDatabaseTables().size(); - return schema; + return MaterializeSchema.fromConnection(getConnection(), getDatabaseName()); } public void addFunctionAndType(String functionName, Character functionType) { From 0eea0212a9a07ec7f6b3f9bf5ae1fe5a101826dd Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Wed, 22 Apr 2026 00:49:07 +0800 Subject: [PATCH 786/856] Fix checkstyle: declare SQLite3UpdateGenerator as final Co-Authored-By: Claude Opus 4.6 --- src/sqlancer/common/gen/AbstractDeleteGenerator.java | 3 +++ src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java index 47df974e6..8dfb0b0c6 100644 --- a/src/sqlancer/common/gen/AbstractDeleteGenerator.java +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -2,4 +2,7 @@ public abstract class AbstractDeleteGenerator extends AbstractGenerator { + protected AbstractDeleteGenerator() { + } + } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 9a2be3150..05f4b9a2e 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -14,7 +14,7 @@ import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; -public class SQLite3UpdateGenerator extends AbstractUpdateGenerator { +public final class SQLite3UpdateGenerator extends AbstractUpdateGenerator { private final SQLite3GlobalState globalState; private final Randomly r; From d4ed473fb7d9ba1237a168144b22b6aeb5c9cf6f Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Apr 2026 10:17:37 +0800 Subject: [PATCH 787/856] Refactor: extract common CREATE INDEX logic into AbstractIndexGenerator Introduce AbstractIndexGenerator with appendCreateIndex(boolean) and appendIndexColumnList(List, boolean) helpers, and convert nine concrete index generators (DuckDB, Doris, Presto, YCQL, TiDB, MariaDB, Materialize, Postgres, YSQL) from static-utility classes to instance-based subclasses whose logic lives in buildStatement(). Public entry points (getQuery/generate) are preserved so callers in the provider classes don't need to change. --- .../common/gen/AbstractIndexGenerator.java | 33 +++++++++++++ .../doris/gen/DorisIndexGenerator.java | 31 ++++++------ .../duckdb/gen/DuckDBIndexGenerator.java | 25 ++++++---- .../mariadb/gen/MariaDBIndexGenerator.java | 48 +++++++------------ .../gen/MaterializeIndexGenerator.java | 24 ++++++---- .../postgres/gen/PostgresIndexGenerator.java | 23 ++++----- .../presto/gen/PrestoIndexGenerator.java | 26 ++++++---- src/sqlancer/tidb/gen/TiDBIndexGenerator.java | 26 +++++----- .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 37 +++++++------- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 23 ++++----- 10 files changed, 167 insertions(+), 129 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractIndexGenerator.java diff --git a/src/sqlancer/common/gen/AbstractIndexGenerator.java b/src/sqlancer/common/gen/AbstractIndexGenerator.java new file mode 100644 index 000000000..bfa62a36c --- /dev/null +++ b/src/sqlancer/common/gen/AbstractIndexGenerator.java @@ -0,0 +1,33 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractIndexGenerator> extends AbstractGenerator { + + protected void appendCreateIndex(boolean unique) { + sb.append("CREATE "); + if (unique) { + sb.append("UNIQUE "); + } + sb.append("INDEX "); + } + + protected void appendIndexColumnList(List columns, boolean allowOrdering) { + sb.append("("); + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + sb.append(columns.get(i).getName()); + if (allowOrdering && Randomly.getBoolean()) { + sb.append(" "); + sb.append(Randomly.fromOptions("ASC", "DESC")); + } + } + sb.append(")"); + } + +} diff --git a/src/sqlancer/doris/gen/DorisIndexGenerator.java b/src/sqlancer/doris/gen/DorisIndexGenerator.java index 5e56bb192..308c09c3f 100644 --- a/src/sqlancer/doris/gen/DorisIndexGenerator.java +++ b/src/sqlancer/doris/gen/DorisIndexGenerator.java @@ -5,42 +5,45 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.doris.DorisProvider.DorisGlobalState; import sqlancer.doris.DorisSchema.DorisColumn; import sqlancer.doris.DorisSchema.DorisTable; -public final class DorisIndexGenerator { +public class DorisIndexGenerator extends AbstractIndexGenerator { - private DorisIndexGenerator() { + private final DorisGlobalState globalState; + + public DorisIndexGenerator(DorisGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(DorisGlobalState globalState) throws SQLException { if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { throw new IgnoreMeException(); } - ExpectedErrors errors = new ExpectedErrors(); + return new DorisIndexGenerator(globalState).getStatement(); + } + @Override + public void buildStatement() { DorisTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - String indexName = globalState.getSchema().getFreeIndexName(); - StringBuilder sb = new StringBuilder("CREATE "); - sb.append("INDEX "); + appendCreateIndex(false); if (Randomly.getBoolean()) { sb.append("IF NOT EXISTS "); } - sb.append(indexName); + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); sb.append(randomTable.getName()); - sb.append("("); - int nr = 1; // Doris Only support CREATE_INDEX on single column and index type is BITMAP; - List subset = Randomly.extractNrRandomColumns(randomTable.getColumns(), nr); - sb.append(subset.get(0).getName()); - sb.append(") "); + // Doris only supports CREATE INDEX on a single column; index type is BITMAP + List subset = Randomly.extractNrRandomColumns(randomTable.getColumns(), 1); + appendIndexColumnList(subset, false); + sb.append(" "); if (Randomly.getBoolean()) { sb.append("USING BITMAP "); } - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 6c50b204d..bd42b64bd 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -3,26 +3,32 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; import sqlancer.duckdb.DuckDBSchema.DuckDBTable; -public final class DuckDBIndexGenerator { +public class DuckDBIndexGenerator extends AbstractIndexGenerator { - private DuckDBIndexGenerator() { + private final DuckDBGlobalState globalState; + + public DuckDBIndexGenerator(DuckDBGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { + return new DuckDBIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Data contains duplicates on indexed column(s)"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); @@ -43,7 +49,6 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("cannot create an index on the rowid"); } - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java index 1ba3fbd4d..1fb6f10b3 100644 --- a/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBIndexGenerator.java @@ -1,31 +1,36 @@ package sqlancer.mariadb.gen; -import java.util.List; - import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mariadb.MariaDBSchema; import sqlancer.mariadb.MariaDBSchema.MariaDBColumn; import sqlancer.mariadb.MariaDBSchema.MariaDBTable; -public final class MariaDBIndexGenerator { +public class MariaDBIndexGenerator extends AbstractIndexGenerator { + + private final MariaDBSchema schema; - private MariaDBIndexGenerator() { + public MariaDBIndexGenerator(MariaDBSchema schema) { + this.schema = schema; + this.canAffectSchema = true; } public static SQLQueryAdapter generate(MariaDBSchema s) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder("CREATE "); + return new MariaDBIndexGenerator(s).getStatement(); + } + + @Override + public void buildStatement() { errors.add("Key/Index cannot be defined on a virtual generated column"); errors.add("Specified key was too long"); - if (Randomly.getBoolean()) { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Duplicate entry"); errors.add("Key/Index cannot be defined on a virtual generated column"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append("i"); sb.append(DBMSCommon.createColumnName(Randomly.smallNumber())); if (Randomly.getBoolean()) { @@ -34,28 +39,9 @@ public static SQLQueryAdapter generate(MariaDBSchema s) { } sb.append(" ON "); - MariaDBTable randomTable = s.getRandomTable(); + MariaDBTable randomTable = schema.getRandomTable(); sb.append(randomTable.getName()); - sb.append("("); - List columns = Randomly.nonEmptySubset(randomTable.getColumns()); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - if (Randomly.getBoolean()) { - sb.append(" "); - sb.append(Randomly.fromOptions("ASC", "DESC")); - } - } - sb.append(")"); - // if (Randomly.getBoolean()) { - // sb.append(" ALGORITHM="); - // sb.append(Randomly.fromOptions("DEFAULT", "INPLACE", "COPY", "NOCOPY", "INSTANT")); - // errors.add("is not supported for this operation"); - // } - - return new SQLQueryAdapter(sb.toString(), errors, true); + appendIndexColumnList(Randomly.nonEmptySubset(randomTable.getColumns()), true); } } diff --git a/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java index 9d7a91b50..a63353f1f 100644 --- a/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeIndexGenerator.java @@ -1,14 +1,18 @@ package sqlancer.materialize.gen; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.materialize.MaterializeGlobalState; +import sqlancer.materialize.MaterializeSchema.MaterializeColumn; import sqlancer.materialize.MaterializeSchema.MaterializeTable; -public final class MaterializeIndexGenerator { +public class MaterializeIndexGenerator extends AbstractIndexGenerator { - private MaterializeIndexGenerator() { + private final MaterializeGlobalState globalState; + + public MaterializeIndexGenerator(MaterializeGlobalState globalState) { + this.globalState = globalState; } public enum IndexType { @@ -16,17 +20,18 @@ public enum IndexType { } public static SQLQueryAdapter generate(MaterializeGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - sb.append(" INDEX "); + return new MaterializeIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(false); MaterializeTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized // views sb.append(MaterializeCommon.getFreeIndexName(globalState.getSchema())); sb.append(" ON "); sb.append(randomTable.getName()); - IndexType method; - method = IndexType.BTREE; + IndexType method = IndexType.BTREE; sb.append("("); if (method == IndexType.HASH) { @@ -75,6 +80,5 @@ public static SQLQueryAdapter generate(MaterializeGlobalState globalState) { errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); MaterializeCommon.addCommonExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } } diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index 4cb2b8e3a..bf70c32d4 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema.PostgresColumn; @@ -15,9 +15,12 @@ import sqlancer.postgres.PostgresVisitor; import sqlancer.postgres.ast.PostgresExpression; -public final class PostgresIndexGenerator { +public class PostgresIndexGenerator extends AbstractIndexGenerator { - private PostgresIndexGenerator() { + private final PostgresGlobalState globalState; + + public PostgresIndexGenerator(PostgresGlobalState globalState) { + this.globalState = globalState; } public enum IndexType { @@ -25,13 +28,12 @@ public enum IndexType { } public static SQLQueryAdapter generate(PostgresGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - if (Randomly.getBoolean()) { - sb.append(" UNIQUE"); - } - sb.append(" INDEX "); + return new PostgresIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(Randomly.getBoolean()); /* * Commented out as a workaround for https://www.postgresql.org/message-id/CA%2Bu7OA4XYhc- * qyCgJqwwgMGZDWAyeH821oa5oMzm_HEifZ4BeA%40mail.gmail.com @@ -136,7 +138,6 @@ public static SQLQueryAdapter generate(PostgresGlobalState globalState) { errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); PostgresCommon.addCommonExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } private static String getNewIndexName(PostgresTable randomTable) { diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java index c76283ee4..5ec23c773 100644 --- a/src/sqlancer/presto/gen/PrestoIndexGenerator.java +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -3,7 +3,7 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema; @@ -12,20 +12,27 @@ import sqlancer.presto.PrestoToStringVisitor; import sqlancer.presto.ast.PrestoExpression; -public final class PrestoIndexGenerator { +public class PrestoIndexGenerator extends AbstractIndexGenerator { - private PrestoIndexGenerator() { + private final PrestoGlobalState globalState; + + public PrestoIndexGenerator(PrestoGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; + this.canonicalizeString = false; } public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { + return new PrestoIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); // cannot query this information sb.append(" ON "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); @@ -50,7 +57,6 @@ public static SQLQueryAdapter getQuery(PrestoGlobalState globalState) { sb.append(PrestoToStringVisitor.asString(expr)); } errors.add("already exists!"); - return new SQLQueryAdapter(sb.toString(), errors, true, false); } } diff --git a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java index 1be2753b6..64b4c808d 100644 --- a/src/sqlancer/tidb/gen/TiDBIndexGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBIndexGenerator.java @@ -5,34 +5,39 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.tidb.TiDBProvider.TiDBGlobalState; import sqlancer.tidb.TiDBSchema.TiDBColumn; import sqlancer.tidb.TiDBSchema.TiDBTable; -public final class TiDBIndexGenerator { +public class TiDBIndexGenerator extends AbstractIndexGenerator { - private TiDBIndexGenerator() { + private final TiDBGlobalState globalState; + + public TiDBIndexGenerator(TiDBGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLException { if (globalState.getSchema().getIndexCount() > globalState.getDbmsSpecificOptions().maxNumIndexes) { throw new IgnoreMeException(); } - ExpectedErrors errors = new ExpectedErrors(); + return new TiDBIndexGenerator(globalState).getStatement(); + } + @Override + public void buildStatement() { TiDBTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); - String indexName = globalState.getSchema().getFreeIndexName(); - StringBuilder sb = new StringBuilder("CREATE "); - if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append("UNIQUE "); + boolean unique = Randomly.getBooleanWithRatherLowProbability(); + if (unique) { errors.add("Duplicate for key"); errors.add("Duplicate entry "); errors.add("A UNIQUE INDEX must include all columns in the table's partitioning function"); } - sb.append("INDEX "); - sb.append(indexName); + appendCreateIndex(unique); + sb.append(globalState.getSchema().getFreeIndexName()); sb.append(" ON "); sb.append(randomTable.getName()); sb.append("("); @@ -63,7 +68,6 @@ public static SQLQueryAdapter getQuery(TiDBGlobalState globalState) throws SQLEx errors.add("index already exist"); errors.add("Data truncation"); errors.add("key was too long"); - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java index ab03316cc..66c59a968 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -1,9 +1,7 @@ package sqlancer.yugabyte.ycql.gen; -import java.util.List; - import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; @@ -11,33 +9,31 @@ import sqlancer.yugabyte.ycql.YCQLToStringVisitor; import sqlancer.yugabyte.ycql.ast.YCQLExpression; -public final class YCQLIndexGenerator { +public class YCQLIndexGenerator extends AbstractIndexGenerator { + + private final YCQLGlobalState globalState; - private YCQLIndexGenerator() { + public YCQLIndexGenerator(YCQLGlobalState globalState) { + this.globalState = globalState; + this.canAffectSchema = true; } public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE "); - if (Randomly.getBoolean()) { + return new YCQLIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + boolean unique = Randomly.getBoolean(); + if (unique) { errors.add("Cant create unique index, table contains duplicate data on indexed column(s)"); - sb.append("UNIQUE "); } - sb.append("INDEX "); + appendCreateIndex(unique); sb.append(Randomly.fromOptions("i0", "i1", "i2", "i3", "i4")); sb.append(" ON "); YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); sb.append(table.getName()); - sb.append("("); - List columns = table.getRandomNonEmptyColumnSubset(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - } - sb.append(")"); + appendIndexColumnList(table.getRandomNonEmptyColumnSubset(), false); if (Randomly.getBoolean()) { sb.append(" WHERE "); YCQLExpression expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) @@ -49,7 +45,6 @@ public static SQLQueryAdapter getQuery(YCQLGlobalState globalState) { errors.add("Invalid CQL Statement"); errors.add( "Invalid Table Definition. Transactions cannot be enabled in an index of a table without transactions enabled."); - return new SQLQueryAdapter(sb.toString(), errors, true); } } diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index 6077dcb1e..ac5b3242f 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractIndexGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ysql.YSQLErrors; @@ -17,19 +17,21 @@ import sqlancer.yugabyte.ysql.YSQLVisitor; import sqlancer.yugabyte.ysql.ast.YSQLExpression; -public final class YSQLIndexGenerator { +public class YSQLIndexGenerator extends AbstractIndexGenerator { - private YSQLIndexGenerator() { + private final YSQLGlobalState globalState; + + public YSQLIndexGenerator(YSQLGlobalState globalState) { + this.globalState = globalState; } public static SQLQueryAdapter generate(YSQLGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - sb.append("CREATE"); - if (Randomly.getBoolean()) { - sb.append(" UNIQUE"); - } - sb.append(" INDEX "); + return new YSQLIndexGenerator(globalState).getStatement(); + } + + @Override + public void buildStatement() { + appendCreateIndex(Randomly.getBoolean()); YSQLTable randomTable = globalState.getSchema().getRandomTable(t -> !t.isView()); // TODO: materialized // views String indexName = getNewIndexName(randomTable); @@ -122,7 +124,6 @@ public static SQLQueryAdapter generate(YSQLGlobalState globalState) { errors.add("result of range difference would not be contiguous"); errors.add("which is part of the partition key"); YSQLErrors.addCommonExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors); } private static String getNewIndexName(YSQLTable randomTable) { From 037e9e7abb2c81179218d34e550507dd9330341e Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Thu, 23 Apr 2026 23:42:25 +0800 Subject: [PATCH 788/856] CI: cache Rust build and pinned tarball downloads Three caches that avoid repeated network work, keyed on pinned versions: - Swatinem/rust-cache for the DataFusion Rust build (cargo registry + target) - actions/cache for the CockroachDB tarball (cockroachdb + cockroachdb-qpg) - actions/cache for the Doris tarball Tarballs are cached as raw .tgz/.tar.gz files rather than extracted dirs, so runtime state from previous runs is not persisted. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ee4186650..9df326bba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,9 +154,15 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true + - name: Cache CockroachDB tarball + uses: actions/cache@v4 + with: + path: cockroach-v24.2.0.linux-amd64.tgz + key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz + tar xzf cockroach-v24.2.0.linux-amd64.tgz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user @@ -180,9 +186,15 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true + - name: Cache CockroachDB tarball + uses: actions/cache@v4 + with: + path: cockroach-v24.2.0.linux-amd64.tgz + key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz + [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz + tar xzf cockroach-v24.2.0.linux-amd64.tgz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user @@ -228,6 +240,10 @@ jobs: with: toolchain: stable override: true + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + workspaces: src/sqlancer/datafusion/server/datafusion_server - name: Build DataFusion Server run: | cd src/sqlancer/datafusion/server/datafusion_server @@ -690,10 +706,15 @@ jobs: run: | sudo apt update sudo apt install mysql-client --assume-yes + - name: Cache Apache Doris tarball + uses: actions/cache@v4 + with: + path: apache-doris-2.1.4-bin-x64.tar.gz + key: apache-doris-2.1.4-bin-x64-tarball - name: Set up Apache Doris run: | sudo sysctl -w vm.max_map_count=2000000 - wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz + [ -f apache-doris-2.1.4-bin-x64.tar.gz ] || wget -q https://apache-doris-releases.oss-accelerate.aliyuncs.com/apache-doris-2.1.4-bin-x64.tar.gz tar zxf apache-doris-2.1.4-bin-x64.tar.gz mv apache-doris-2.1.4-bin-x64 apache-doris sudo swapoff -a From ee60e918d06f0f11105a9182a9852a6fa1d3209c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 10:20:56 +0800 Subject: [PATCH 789/856] CI: drop CockroachDB tarball cache (no measured benefit) Warm-cache rerun showed CockroachDB setup at 11s vs 10s baseline - binaries.cockroachdb.com is fast enough that cache restore offers no saving. Keep the Rust build (-90s) and Doris (-216s) caches. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/main.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9df326bba..e85c5c219 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -154,15 +154,9 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Cache CockroachDB tarball - uses: actions/cache@v4 - with: - path: cockroach-v24.2.0.linux-amd64.tgz - key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz - tar xzf cockroach-v24.2.0.linux-amd64.tgz + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user @@ -186,15 +180,9 @@ jobs: cache: 'maven' - name: Build SQLancer run: mvn -B package -DskipTests=true - - name: Cache CockroachDB tarball - uses: actions/cache@v4 - with: - path: cockroach-v24.2.0.linux-amd64.tgz - key: cockroach-v24.2.0-linux-amd64-tgz - name: Set up CockroachDB run: | - [ -f cockroach-v24.2.0.linux-amd64.tgz ] || wget -q https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz - tar xzf cockroach-v24.2.0.linux-amd64.tgz + wget -qO- https://binaries.cockroachdb.com/cockroach-v24.2.0.linux-amd64.tgz | tar xvz cd cockroach-v24.2.0.linux-amd64/ && ./cockroach start-single-node --insecure & until cockroach-v24.2.0.linux-amd64/cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done - name: Create SQLancer user From b2df941f170bb87a975b3a55cc14692ef9b68514 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 00:07:52 +0800 Subject: [PATCH 790/856] Refactor: extract common CREATE TABLE logic into AbstractTableGenerator Introduce AbstractTableGenerator with appendCreateTable header helpers and appendColumnDefinitions / appendColumnDefinitionList iteration helpers that delegate to an overridable appendColumnDefinition(C) (default: "name type"). Convert four concrete table generators (QuestDB, HSQLDB, Databend, DuckDB) from plain classes to instance-based subclasses whose logic lives in buildStatement(). Public entry points (getQuery) are preserved so callers in the provider classes don't need to change. Generators with DB-specific shape (temporary/unlogged modifiers, in-place column building, partitioning, custom return types, etc.) are left alone. --- .../common/gen/AbstractTableGenerator.java | 42 +++++++++ .../databend/gen/DatabendTableGenerator.java | 65 +++++++------ .../duckdb/gen/DuckDBTableGenerator.java | 92 ++++++++++--------- .../hsqldb/gen/HSQLDBTableGenerator.java | 60 ++++++------ .../questdb/gen/QuestDBTableGenerator.java | 41 ++++----- 5 files changed, 178 insertions(+), 122 deletions(-) create mode 100644 src/sqlancer/common/gen/AbstractTableGenerator.java diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java new file mode 100644 index 000000000..e1cf34832 --- /dev/null +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -0,0 +1,42 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.schema.AbstractTableColumn; + +public abstract class AbstractTableGenerator> extends AbstractGenerator { + + protected void appendCreateTable(String tableName) { + appendCreateTable(tableName, false); + } + + protected void appendCreateTable(String tableName, boolean ifNotExists) { + sb.append("CREATE TABLE "); + if (ifNotExists) { + sb.append("IF NOT EXISTS "); + } + sb.append(tableName); + } + + protected void appendColumnDefinitions(List columns) { + sb.append("("); + appendColumnDefinitionList(columns); + sb.append(")"); + } + + protected void appendColumnDefinitionList(List columns) { + for (int i = 0; i < columns.size(); i++) { + if (i != 0) { + sb.append(", "); + } + appendColumnDefinition(columns.get(i)); + } + } + + protected void appendColumnDefinition(C column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + } + +} diff --git a/src/sqlancer/databend/gen/DatabendTableGenerator.java b/src/sqlancer/databend/gen/DatabendTableGenerator.java index 514d740d0..2c3416538 100644 --- a/src/sqlancer/databend/gen/DatabendTableGenerator.java +++ b/src/sqlancer/databend/gen/DatabendTableGenerator.java @@ -4,8 +4,8 @@ import java.util.List; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.gen.TypedExpressionGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; @@ -15,44 +15,49 @@ import sqlancer.databend.DatabendToStringVisitor; import sqlancer.databend.ast.DatabendExpression; -public class DatabendTableGenerator { +public class DatabendTableGenerator extends AbstractTableGenerator { + + private DatabendGlobalState globalState; + private TypedExpressionGenerator gen; + + public DatabendTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(DatabendGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { DatabendErrors.addExpressionErrors(errors); - StringBuilder sb = new StringBuilder(); String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName); List columns = getNewColumns(); - TypedExpressionGenerator gen = new DatabendNewExpressionGenerator( - globalState).setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); + gen = new DatabendNewExpressionGenerator(globalState).setColumns(columns); + appendColumnDefinitions(columns); + } - if (globalState.getDbmsSpecificOptions().testNotNullConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" NOT NULL"); - } else { - sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 - } + @Override + protected void appendColumnDefinition(DatabendColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); - if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { - sb.append(" DEFAULT("); - sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 - gen.generateConstant(columns.get(i).getType().getPrimitiveDataType()))); - sb.append(")"); - } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } else { + sb.append(" NULL"); // Databend 默认字段为非空，这个将它默认设置为允许空 } - sb.append(")"); - return new SQLQueryAdapter(sb.toString(), errors, true); + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DatabendToStringVisitor.asString(// 常量类型于字段类型等同 + gen.generateConstant(column.getType().getPrimitiveDataType()))); + sb.append(")"); + } } private static List getNewColumns() { diff --git a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java index ea6d3537f..c8ac6f28d 100644 --- a/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBTableGenerator.java @@ -5,8 +5,8 @@ import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; -import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.duckdb.DuckDBErrors; import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; @@ -16,50 +16,28 @@ import sqlancer.duckdb.DuckDBToStringVisitor; import sqlancer.duckdb.ast.DuckDBExpression; -public class DuckDBTableGenerator { +public class DuckDBTableGenerator extends AbstractTableGenerator { + + private DuckDBGlobalState globalState; + private UntypedExpressionGenerator gen; + + public DuckDBTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName); List columns = getNewColumns(); - UntypedExpressionGenerator gen = new DuckDBExpressionGenerator(globalState) - .setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() - && columns.get(i).getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { - sb.append(" COLLATE "); - sb.append(getRandomCollate()); - } - if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" UNIQUE"); - } - if (globalState.getDbmsSpecificOptions().testNotNullConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" NOT NULL"); - } - if (globalState.getDbmsSpecificOptions().testCheckConstraints - && Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" CHECK("); - sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); - DuckDBErrors.addExpressionErrors(errors); - sb.append(")"); - } - if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { - sb.append(" DEFAULT("); - sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); - sb.append(")"); - } - } + gen = new DuckDBExpressionGenerator(globalState).setColumns(columns); + sb.append("("); + appendColumnDefinitionList(columns); if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { errors.add("Invalid type for index"); List primaryKeyColumns = Randomly.nonEmptySubset(columns); @@ -68,7 +46,37 @@ public SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { sb.append(")"); } sb.append(")"); - return new SQLQueryAdapter(sb.toString(), errors, true); + } + + @Override + protected void appendColumnDefinition(DuckDBColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType()); + if (globalState.getDbmsSpecificOptions().testCollate && Randomly.getBooleanWithRatherLowProbability() + && column.getType().getPrimitiveDataType() == DuckDBDataType.VARCHAR) { + sb.append(" COLLATE "); + sb.append(getRandomCollate()); + } + if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" UNIQUE"); + } + if (globalState.getDbmsSpecificOptions().testNotNullConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" NOT NULL"); + } + if (globalState.getDbmsSpecificOptions().testCheckConstraints + && Randomly.getBooleanWithRatherLowProbability()) { + sb.append(" CHECK("); + sb.append(DuckDBToStringVisitor.asString(gen.generateExpression())); + DuckDBErrors.addExpressionErrors(errors); + sb.append(")"); + } + if (Randomly.getBoolean() && globalState.getDbmsSpecificOptions().testDefaultValues) { + sb.append(" DEFAULT("); + sb.append(DuckDBToStringVisitor.asString(gen.generateConstant())); + sb.append(")"); + } } public static String getRandomCollate() { diff --git a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java index 48606e9bf..30249e780 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBTableGenerator.java @@ -5,44 +5,48 @@ import javax.annotation.Nullable; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.hsqldb.HSQLDBProvider; import sqlancer.hsqldb.HSQLDBSchema; -public class HSQLDBTableGenerator { +public class HSQLDBTableGenerator extends AbstractTableGenerator { + + private HSQLDBProvider.HSQLDBGlobalState globalState; + private String tableName; + + public HSQLDBTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(HSQLDBProvider.HSQLDBGlobalState globalState, @Nullable String tableName) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + this.tableName = tableName; + return getStatement(); + } + + @Override + public void buildStatement() { String name = tableName; - if (tableName == null) { + if (name == null) { name = globalState.getSchema().getFreeTableName(); } - sb.append("CREATE TABLE "); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(name); - sb.append("("); - List columns = getNewColumns(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType().getType().name()); - if (columns.get(i).getType().getSize() > 0) { - // Cannot specify size for non composite data types - sb.append("("); - sb.append(columns.get(i).getType().getSize()); - sb.append(")"); - } - } - sb.append(")"); + appendCreateTable(name, Randomly.getBoolean()); + appendColumnDefinitions(getNewColumns()); sb.append(";"); - return new SQLQueryAdapter(sb.toString(), errors, true); + } + + @Override + protected void appendColumnDefinition(HSQLDBSchema.HSQLDBColumn column) { + sb.append(column.getName()); + sb.append(" "); + sb.append(column.getType().getType().name()); + if (column.getType().getSize() > 0) { + // Cannot specify size for non composite data types + sb.append("("); + sb.append(column.getType().getSize()); + sb.append(")"); + } } private static List getNewColumns() { diff --git a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java index a308b8d10..d4b17e76b 100644 --- a/src/sqlancer/questdb/gen/QuestDBTableGenerator.java +++ b/src/sqlancer/questdb/gen/QuestDBTableGenerator.java @@ -5,40 +5,37 @@ import javax.annotation.Nullable; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.questdb.QuestDBProvider.QuestDBGlobalState; import sqlancer.questdb.QuestDBSchema.QuestDBColumn; import sqlancer.questdb.QuestDBSchema.QuestDBCompositeDataType; -public class QuestDBTableGenerator { +public class QuestDBTableGenerator extends AbstractTableGenerator { + + private QuestDBGlobalState globalState; + private String tableName; + + public QuestDBTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(QuestDBGlobalState globalState, @Nullable String tableName) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + this.tableName = tableName; + return getStatement(); + } + + @Override + public void buildStatement() { String name = tableName; - if (tableName == null) { + if (name == null) { name = globalState.getSchema().getFreeTableName(); } - sb.append("CREATE TABLE "); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(name); - sb.append("("); - List columns = getNewColumns(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - } - sb.append(")"); + appendCreateTable(name, Randomly.getBoolean()); + appendColumnDefinitions(getNewColumns()); sb.append(";"); errors.add("table already exists"); - return new SQLQueryAdapter(sb.toString(), errors, true); } private static List getNewColumns() { From 8d53aabc5080095dc0350e452ed1bee8f04be149 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 00:24:33 +0800 Subject: [PATCH 791/856] Refactor: migrate Presto, YCQL, Spark table generators to AbstractTableGenerator Follow-up to the initial four migrations. Presto and YCQL fit the canonical CREATE TABLE [IF NOT EXISTS] (col type[, ...]) shape directly. Spark needs the columns pre-built before iteration so that appendColumnDefinitions can call back into the overridden appendColumnDefinition; this is safe because Spark's DEFAULT clause uses generateConstant, which does not reference other columns. Hive is left alone: its CHECK/DEFAULT constraints use generateExpression which reads the columnsToBeAdded list incrementally, so pre-building would change expression-generation behavior. --- .../presto/gen/PrestoTableGenerator.java | 70 +++++++------------ .../spark/gen/SparkTableGenerator.java | 45 +++++------- .../yugabyte/ycql/gen/YCQLTableGenerator.java | 37 +++++----- 3 files changed, 60 insertions(+), 92 deletions(-) diff --git a/src/sqlancer/presto/gen/PrestoTableGenerator.java b/src/sqlancer/presto/gen/PrestoTableGenerator.java index 1d7df2ee6..49e1346ad 100644 --- a/src/sqlancer/presto/gen/PrestoTableGenerator.java +++ b/src/sqlancer/presto/gen/PrestoTableGenerator.java @@ -4,13 +4,35 @@ import java.util.List; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.presto.PrestoGlobalState; import sqlancer.presto.PrestoSchema.PrestoColumn; import sqlancer.presto.PrestoSchema.PrestoCompositeDataType; -public class PrestoTableGenerator { +public class PrestoTableGenerator extends AbstractTableGenerator { + + private PrestoGlobalState globalState; + + public PrestoTableGenerator() { + this.canAffectSchema = true; + this.canonicalizeString = false; + } + + public SQLQueryAdapter getQuery(PrestoGlobalState globalState) { + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { + String catalog = globalState.getDbmsSpecificOptions().catalog; + String schema = globalState.getDatabaseName(); + String tableName = globalState.getSchema().getFreeTableName(); + String qualifiedName = catalog + "." + schema + "." + tableName; + appendCreateTable(qualifiedName); + appendColumnDefinitions(getNewColumns()); + } private static List getNewColumns() { List columns = new ArrayList<>(); @@ -22,48 +44,4 @@ private static List getNewColumns() { return columns; } - public SQLQueryAdapter getQuery(PrestoGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); - String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - String catalog = globalState.getDbmsSpecificOptions().catalog; - String schema = globalState.getDatabaseName(); - - sb.append(catalog).append("."); - sb.append(schema).append("."); - - sb.append(tableName); - sb.append("("); - List columns = getNewColumns(); - // TypedExpressionGenerator, PrestoColumn, PrestoCompositeDataType> - // typedExpressionGenerator = new PrestoTypedExpressionGenerator(globalState).setColumns(columns); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - PrestoColumn column = columns.get(i); - sb.append(column.getName()); - sb.append(" "); - sb.append(column.getType()); - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" UNIQUE"); - // } - // if (globalState.getDbmsSpecificOptions().testNotNullConstraints - // && Randomly.getBooleanWithRatherLowProbability()) { - // sb.append(" NOT NULL"); - // } - } - // if (globalState.getDbmsSpecificOptions().testIndexes && Randomly.getBoolean()) { - // errors.add("Invalid type for index"); - // List primaryKeyColumns = Randomly.nonEmptySubset(columns); - // sb.append(", PRIMARY KEY("); - // sb.append(primaryKeyColumns.stream().map(c -> c.getName()).collect(Collectors.joining(", "))); - // sb.append(")"); - // } - sb.append(")"); - - return new SQLQueryAdapter(sb.toString(), errors, true, false); - } - } diff --git a/src/sqlancer/spark/gen/SparkTableGenerator.java b/src/sqlancer/spark/gen/SparkTableGenerator.java index 937e52248..2c26ea34c 100644 --- a/src/sqlancer/spark/gen/SparkTableGenerator.java +++ b/src/sqlancer/spark/gen/SparkTableGenerator.java @@ -5,7 +5,7 @@ import sqlancer.Randomly; import sqlancer.common.DBMSCommon; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.spark.SparkErrors; import sqlancer.spark.SparkGlobalState; @@ -15,7 +15,7 @@ import sqlancer.spark.SparkSchema.SparkTable; import sqlancer.spark.SparkToStringVisitor; -public class SparkTableGenerator { +public class SparkTableGenerator extends AbstractTableGenerator { private enum ColumnConstraints { NOT_NULL, DEFAULT @@ -27,7 +27,6 @@ private enum ColumnConstraints { private final SparkGlobalState globalState; private final String tableName; - private final StringBuilder sb = new StringBuilder(); private final SparkExpressionGenerator gen; private final SparkTable table; private final List columnsToBeAdded = new ArrayList<>(); @@ -37,28 +36,25 @@ public SparkTableGenerator(SparkGlobalState globalState, String tableName) { this.globalState = globalState; this.table = new SparkTable(tableName, columnsToBeAdded, false); this.gen = new SparkExpressionGenerator(globalState).setColumns(columnsToBeAdded); + this.canAffectSchema = true; + this.canonicalizeString = false; } public static SQLQueryAdapter generate(SparkGlobalState globalState, String tableName) { - SparkTableGenerator generator = new SparkTableGenerator(globalState, tableName); - return generator.create(); + return new SparkTableGenerator(globalState, tableName).getStatement(); } - private SQLQueryAdapter create() { - ExpectedErrors errors = new ExpectedErrors(); - - sb.append("CREATE TABLE "); - sb.append(globalState.getDatabaseName()); - sb.append("."); - sb.append(tableName); - sb.append(" ("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - if (i != 0) { - sb.append(", "); - } - appendColumn(i); + @Override + public void buildStatement() { + int columnCount = Randomly.smallNumber() + 1; + for (int i = 0; i < columnCount; i++) { + String columnName = DBMSCommon.createColumnName(i); + SparkDataType type = SparkSchema.SparkDataType.getRandomType(); + columnsToBeAdded.add(new SparkColumn(columnName, table, type)); } - sb.append(")"); + appendCreateTable(globalState.getDatabaseName() + "." + tableName); + sb.append(" "); + appendColumnDefinitions(columnsToBeAdded); sb.append(" USING PARQUET"); // TODO: implement PARTITION BY clause @@ -67,16 +63,13 @@ private SQLQueryAdapter create() { // TODO: randomly add some predefined TABLEPROPERTIES SparkErrors.addExpressionErrors(errors); - return new SQLQueryAdapter(sb.toString(), errors, true, false); } - private void appendColumn(int columnId) { - String columnName = DBMSCommon.createColumnName(columnId); - sb.append(columnName); + @Override + protected void appendColumnDefinition(SparkColumn column) { + sb.append(column.getName()); sb.append(" "); - SparkDataType randType = SparkSchema.SparkDataType.getRandomType(); - sb.append(randType); - columnsToBeAdded.add(new SparkColumn(columnName, table, randType)); + sb.append(column.getType()); appendColumnConstraint(); } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java index 148a1cdbc..c57f94287 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLTableGenerator.java @@ -5,35 +5,33 @@ import java.util.stream.Collectors; import sqlancer.Randomly; -import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.gen.AbstractTableGenerator; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.yugabyte.ycql.YCQLProvider.YCQLGlobalState; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLColumn; import sqlancer.yugabyte.ycql.YCQLSchema.YCQLCompositeDataType; -public class YCQLTableGenerator { +public class YCQLTableGenerator extends AbstractTableGenerator { + + private YCQLGlobalState globalState; + + public YCQLTableGenerator() { + this.canAffectSchema = true; + } public SQLQueryAdapter getQuery(YCQLGlobalState globalState) { - ExpectedErrors errors = new ExpectedErrors(); - StringBuilder sb = new StringBuilder(); + this.globalState = globalState; + return getStatement(); + } + + @Override + public void buildStatement() { String tableName = globalState.getSchema().getFreeTableName(); - sb.append("CREATE TABLE "); - if (Randomly.getBoolean()) { - sb.append("IF NOT EXISTS "); - } - sb.append(tableName); - sb.append("("); + appendCreateTable(tableName, Randomly.getBoolean()); List columns = getNewColumns(); - for (int i = 0; i < columns.size(); i++) { - if (i != 0) { - sb.append(", "); - } - sb.append(columns.get(i).getName()); - sb.append(" "); - sb.append(columns.get(i).getType()); - // todo PK, STATIC - } + sb.append("("); + appendColumnDefinitionList(columns); errors.add("Query timed out after PT2S"); errors.add("Invalid type for index"); List primaryKeyColumns = Randomly.nonEmptySubset(columns); @@ -41,7 +39,6 @@ public SQLQueryAdapter getQuery(YCQLGlobalState globalState) { sb.append(primaryKeyColumns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); sb.append(")"); sb.append(")"); - return new SQLQueryAdapter(sb.toString(), errors, true); } private static List getNewColumns() { From db0d9e62bc82abfa387272a969b20bf5061739a6 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Fri, 24 Apr 2026 00:31:33 +0800 Subject: [PATCH 792/856] Document AbstractTableGenerator helpers with example output Add short Javadoc to each helper showing what SQL fragment it emits, so subclass authors can pick the right entry point at a glance. --- .../common/gen/AbstractTableGenerator.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java index e1cf34832..c6ca55cbc 100644 --- a/src/sqlancer/common/gen/AbstractTableGenerator.java +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -6,10 +6,12 @@ public abstract class AbstractTableGenerator> extends AbstractGenerator { + /** Appends {@code CREATE TABLE }. */ protected void appendCreateTable(String tableName) { appendCreateTable(tableName, false); } + /** Appends {@code CREATE TABLE [IF NOT EXISTS ]}. */ protected void appendCreateTable(String tableName, boolean ifNotExists) { sb.append("CREATE TABLE "); if (ifNotExists) { @@ -18,12 +20,20 @@ protected void appendCreateTable(String tableName, boolean ifNotExists) { sb.append(tableName); } + /** + * Appends a parenthesized, comma-separated column definition list, e.g. {@code (c0 INT, c1 TEXT)}. Delegates each + * column's rendering to {@link #appendColumnDefinition(AbstractTableColumn)}. + */ protected void appendColumnDefinitions(List columns) { sb.append("("); appendColumnDefinitionList(columns); sb.append(")"); } + /** + * Appends a comma-separated column definition list without enclosing parentheses, e.g. {@code c0 INT, c1 TEXT}. + * Useful when subclasses also emit table-level constraints (e.g. {@code PRIMARY KEY (...)}) inside the same parens. + */ protected void appendColumnDefinitionList(List columns) { for (int i = 0; i < columns.size(); i++) { if (i != 0) { @@ -33,6 +43,10 @@ protected void appendColumnDefinitionList(List columns) { } } + /** + * Appends a single column's definition. Default output is {@code }, e.g. {@code c0 INT}. Override to + * add constraints such as {@code NOT NULL}, {@code DEFAULT ...}, or {@code CHECK (...)}. + */ protected void appendColumnDefinition(C column) { sb.append(column.getName()); sb.append(" "); From 220835e5fb96383cf814ba9407e1891e282a528c Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sat, 25 Apr 2026 23:30:28 +0800 Subject: [PATCH 793/856] Fix checkstyle: add @param tags to AbstractTableGenerator Javadoc Co-Authored-By: Claude Opus 4.7 --- .../common/gen/AbstractTableGenerator.java | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/common/gen/AbstractTableGenerator.java b/src/sqlancer/common/gen/AbstractTableGenerator.java index c6ca55cbc..c8d1d9697 100644 --- a/src/sqlancer/common/gen/AbstractTableGenerator.java +++ b/src/sqlancer/common/gen/AbstractTableGenerator.java @@ -6,12 +6,24 @@ public abstract class AbstractTableGenerator> extends AbstractGenerator { - /** Appends {@code CREATE TABLE }. */ + /** + * Appends {@code CREATE TABLE }. + * + * @param tableName + * the name of the table to create. + */ protected void appendCreateTable(String tableName) { appendCreateTable(tableName, false); } - /** Appends {@code CREATE TABLE [IF NOT EXISTS ]}. */ + /** + * Appends {@code CREATE TABLE [IF NOT EXISTS ]}. + * + * @param tableName + * the name of the table to create. + * @param ifNotExists + * whether to emit the {@code IF NOT EXISTS} clause. + */ protected void appendCreateTable(String tableName, boolean ifNotExists) { sb.append("CREATE TABLE "); if (ifNotExists) { @@ -23,6 +35,9 @@ protected void appendCreateTable(String tableName, boolean ifNotExists) { /** * Appends a parenthesized, comma-separated column definition list, e.g. {@code (c0 INT, c1 TEXT)}. Delegates each * column's rendering to {@link #appendColumnDefinition(AbstractTableColumn)}. + * + * @param columns + * the columns to render. */ protected void appendColumnDefinitions(List columns) { sb.append("("); @@ -33,6 +48,9 @@ protected void appendColumnDefinitions(List columns) { /** * Appends a comma-separated column definition list without enclosing parentheses, e.g. {@code c0 INT, c1 TEXT}. * Useful when subclasses also emit table-level constraints (e.g. {@code PRIMARY KEY (...)}) inside the same parens. + * + * @param columns + * the columns to render. */ protected void appendColumnDefinitionList(List columns) { for (int i = 0; i < columns.size(); i++) { @@ -46,6 +64,9 @@ protected void appendColumnDefinitionList(List columns) { /** * Appends a single column's definition. Default output is {@code }, e.g. {@code c0 INT}. Override to * add constraints such as {@code NOT NULL}, {@code DEFAULT ...}, or {@code CHECK (...)}. + * + * @param column + * the column whose definition to render. */ protected void appendColumnDefinition(C column) { sb.append(column.getName()); From 12048b67e9134820acc98037221fd1b41e1867c1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Sun, 26 Apr 2026 00:16:31 +0800 Subject: [PATCH 794/856] Refactor: extract common DELETE generation logic into AbstractDeleteGenerator Add appendDeleteFromTable, appendLimitClause, and appendReturningClause helpers in AbstractDeleteGenerator and migrate all 15 DELETE generators that extend it. Lift the WHERE-clause helper to AbstractGenerator since it is also used by UPDATE, partial-INDEX, and INSERT...ON CONFLICT generators, and migrate 17 of those generators to use it. Co-Authored-By: Claude Opus 4.7 --- .../gen/CockroachDBDeleteGenerator.java | 6 +-- .../gen/CockroachDBUpdateGenerator.java | 3 +- .../common/gen/AbstractDeleteGenerator.java | 50 +++++++++++++++++++ .../common/gen/AbstractGenerator.java | 13 +++++ .../databend/gen/DatabendDeleteGenerator.java | 6 +-- .../doris/gen/DorisDeleteGenerator.java | 6 +-- .../doris/gen/DorisUpdateGenerator.java | 3 +- .../duckdb/gen/DuckDBDeleteGenerator.java | 6 +-- src/sqlancer/h2/H2DeleteGenerator.java | 9 ++-- src/sqlancer/h2/H2UpdateGenerator.java | 3 +- .../hsqldb/gen/HSQLDBUpdateGenerator.java | 3 +- .../mariadb/gen/MariaDBDeleteGenerator.java | 17 ++++--- .../gen/MaterializeDeleteGenerator.java | 7 +-- .../gen/MaterializeUpdateGenerator.java | 3 +- .../mysql/gen/MySQLDeleteGenerator.java | 3 +- .../mysql/gen/MySQLUpdateGenerator.java | 3 +- .../gen/OceanBaseDeleteGenerator.java | 3 +- .../gen/OceanBaseUpdateGenerator.java | 3 +- .../postgres/gen/PostgresDeleteGenerator.java | 13 ++--- .../postgres/gen/PostgresIndexGenerator.java | 3 +- .../postgres/gen/PostgresUpdateGenerator.java | 3 +- .../presto/gen/PrestoDeleteGenerator.java | 6 +-- .../presto/gen/PrestoIndexGenerator.java | 3 +- .../gen/dml/SQLite3DeleteGenerator.java | 6 +-- .../gen/dml/SQLite3UpdateGenerator.java | 3 +- .../tidb/gen/TiDBDeleteGenerator.java | 6 +-- .../tidb/gen/TiDBUpdateGenerator.java | 3 +- .../ycql/gen/YCQLDeleteGenerator.java | 6 +-- .../yugabyte/ycql/gen/YCQLIndexGenerator.java | 3 +- .../ysql/gen/YSQLDeleteGenerator.java | 13 ++--- .../yugabyte/ysql/gen/YSQLIndexGenerator.java | 3 +- .../ysql/gen/YSQLUpdateGenerator.java | 3 +- 32 files changed, 116 insertions(+), 105 deletions(-) diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java index f0048d278..dd1e94aa9 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBDeleteGenerator.java @@ -24,12 +24,10 @@ public static SQLQueryAdapter delete(CockroachDBGlobalState globalState) { @Override public void buildStatement() { CockroachDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); CockroachDBErrors.addExpressionErrors(errors); - sb.append(CockroachDBVisitor.asString(new CockroachDBExpressionGenerator(globalState) + appendWhereClause(CockroachDBVisitor.asString(new CockroachDBExpressionGenerator(globalState) .setColumns(table.getColumns()).generateExpression(CockroachDBDataType.BOOL.get()))); } else { errors.add("rejected: DELETE without WHERE clause (sql_safe_updates = true)"); diff --git a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java index b367f5c59..06cc0f8ee 100644 --- a/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java +++ b/src/sqlancer/cockroachdb/gen/CockroachDBUpdateGenerator.java @@ -40,8 +40,7 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); + appendWhereClause(CockroachDBVisitor.asString(gen.generateExpression(CockroachDBDataType.BOOL.get()))); } errors.add("violates unique constraint"); errors.add("violates not-null constraint"); diff --git a/src/sqlancer/common/gen/AbstractDeleteGenerator.java b/src/sqlancer/common/gen/AbstractDeleteGenerator.java index 8dfb0b0c6..f3b5a1955 100644 --- a/src/sqlancer/common/gen/AbstractDeleteGenerator.java +++ b/src/sqlancer/common/gen/AbstractDeleteGenerator.java @@ -5,4 +5,54 @@ public abstract class AbstractDeleteGenerator extends AbstractGenerator { protected AbstractDeleteGenerator() { } + /** + * Appends {@code DELETE FROM }. + * + * @param tableName + * the name of the table to delete from. + */ + protected void appendDeleteFromTable(String tableName) { + appendDeleteFromTable(tableName, false); + } + + /** + * Appends {@code DELETE FROM [ONLY ]}. + * + * @param tableName + * the name of the table to delete from. + * @param only + * whether to emit the {@code ONLY} keyword (used by some databases to restrict deletion to the named + * table rather than its inheritance descendants). + */ + protected void appendDeleteFromTable(String tableName, boolean only) { + sb.append("DELETE FROM "); + if (only) { + sb.append("ONLY "); + } + sb.append(tableName); + } + + /** + * Appends {@code LIMIT } (with a leading space). + * + * @param value + * the LIMIT value, e.g. an integer literal or already-rendered expression. Converted via + * {@link StringBuilder#append(Object)}. + */ + protected void appendLimitClause(Object value) { + sb.append(" LIMIT "); + sb.append(value); + } + + /** + * Appends {@code RETURNING } (with a leading space). + * + * @param expression + * the rendered RETURNING expression. + */ + protected void appendReturningClause(String expression) { + sb.append(" RETURNING "); + sb.append(expression); + } + } diff --git a/src/sqlancer/common/gen/AbstractGenerator.java b/src/sqlancer/common/gen/AbstractGenerator.java index 5d13fc746..dbdf100d5 100644 --- a/src/sqlancer/common/gen/AbstractGenerator.java +++ b/src/sqlancer/common/gen/AbstractGenerator.java @@ -17,4 +17,17 @@ public SQLQueryAdapter getStatement() { public abstract void buildStatement(); + /** + * Appends {@code WHERE } (with a leading space). Subclasses are responsible for deciding whether to + * include the WHERE clause, typically based on a randomized boolean. Used by DELETE, UPDATE, partial-INDEX, and + * INSERT...ON CONFLICT generators. + * + * @param condition + * the rendered WHERE condition. + */ + protected void appendWhereClause(String condition) { + sb.append(" WHERE "); + sb.append(condition); + } + } diff --git a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java index 5adabf175..22336fd72 100644 --- a/src/sqlancer/databend/gen/DatabendDeleteGenerator.java +++ b/src/sqlancer/databend/gen/DatabendDeleteGenerator.java @@ -22,11 +22,9 @@ public static SQLQueryAdapter generate(DatabendGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); - sb.append(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); + appendDeleteFromTable(globalState.getSchema().getRandomTable(t -> !t.isView()).getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DatabendToStringVisitor.asString( + appendWhereClause(DatabendToStringVisitor.asString( new DatabendNewExpressionGenerator(globalState).generateExpression(DatabendDataType.BOOLEAN))); DatabendErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/doris/gen/DorisDeleteGenerator.java b/src/sqlancer/doris/gen/DorisDeleteGenerator.java index 66deaea8f..b155e0381 100644 --- a/src/sqlancer/doris/gen/DorisDeleteGenerator.java +++ b/src/sqlancer/doris/gen/DorisDeleteGenerator.java @@ -23,12 +23,10 @@ public static SQLQueryAdapter generate(DorisGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); DorisTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) + appendWhereClause(DorisToStringVisitor.asString(new DorisNewExpressionGenerator(globalState) .setColumns(table.getColumns()).generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addExpressionErrors(errors); } diff --git a/src/sqlancer/doris/gen/DorisUpdateGenerator.java b/src/sqlancer/doris/gen/DorisUpdateGenerator.java index e0db2b7b4..93f835eff 100644 --- a/src/sqlancer/doris/gen/DorisUpdateGenerator.java +++ b/src/sqlancer/doris/gen/DorisUpdateGenerator.java @@ -35,8 +35,7 @@ public void buildStatement() { sb.append(table.getName()); sb.append(" SET "); updateColumns(columns); - sb.append(" WHERE "); - sb.append(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); + appendWhereClause(DorisToStringVisitor.asString(gen.generateExpression(DorisSchema.DorisDataType.BOOLEAN))); DorisErrors.addInsertErrors(errors); } diff --git a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java index 42695a9f6..9f4f4ed6a 100644 --- a/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBDeleteGenerator.java @@ -22,12 +22,10 @@ public static SQLQueryAdapter generate(DuckDBGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); DuckDBTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(DuckDBToStringVisitor.asString( + appendWhereClause(DuckDBToStringVisitor.asString( new DuckDBExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } DuckDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/h2/H2DeleteGenerator.java b/src/sqlancer/h2/H2DeleteGenerator.java index dd0fdff34..291ee99c2 100644 --- a/src/sqlancer/h2/H2DeleteGenerator.java +++ b/src/sqlancer/h2/H2DeleteGenerator.java @@ -20,17 +20,14 @@ public static SQLQueryAdapter getQuery(H2GlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); H2Table table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(H2ToStringVisitor.asString( + appendWhereClause(H2ToStringVisitor.asString( new H2ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } if (Randomly.getBoolean()) { - sb.append(" LIMIT "); - sb.append(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); + appendLimitClause(H2ToStringVisitor.asString(new H2ExpressionGenerator(globalState).generateConstant())); } H2Errors.addExpressionErrors(errors); H2Errors.addDeleteErrors(errors); diff --git a/src/sqlancer/h2/H2UpdateGenerator.java b/src/sqlancer/h2/H2UpdateGenerator.java index fe63c7a4e..05e348038 100644 --- a/src/sqlancer/h2/H2UpdateGenerator.java +++ b/src/sqlancer/h2/H2UpdateGenerator.java @@ -34,8 +34,7 @@ public void buildStatement() { H2Errors.addInsertErrors(errors); H2Errors.addDeleteErrors(errors); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(H2ToStringVisitor.asString(gen.generateExpression())); + appendWhereClause(H2ToStringVisitor.asString(gen.generateExpression())); } H2Errors.addExpressionErrors(errors); } diff --git a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java index 54380214f..2e6081df4 100644 --- a/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java +++ b/src/sqlancer/hsqldb/gen/HSQLDBUpdateGenerator.java @@ -37,8 +37,7 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); - sb.append(HSQLDBToStringVisitor.asString( + appendWhereClause(HSQLDBToStringVisitor.asString( gen.generateExpression(HSQLDBCompositeDataType.getRandomWithType(HSQLDBDataType.BOOLEAN)))); errors.add("data type of expression is not boolean"); HSQLDBErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java index 2992f569d..da1651662 100644 --- a/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java +++ b/src/sqlancer/mariadb/gen/MariaDBDeleteGenerator.java @@ -57,12 +57,13 @@ public void buildStatement() { sb.append(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); + String condition; if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); + condition = MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r)); } else { - sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); + condition = MariaDBVisitor.asString(expressionGenerator.getRandomExpression()); } + appendWhereClause(condition); } // ORDER BY + LIMIT @@ -75,18 +76,18 @@ public void buildStatement() { } if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" LIMIT "); - sb.append(Randomly.getNotCachedInteger(1, 10)); + appendLimitClause(Randomly.getNotCachedInteger(1, 10)); } // RETURNING clause (MariaDB >= 10.5) if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(" RETURNING "); + String expression; if (Randomly.getBooleanWithRatherLowProbability()) { - sb.append(MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r))); + expression = MariaDBVisitor.asString(MariaDBExpressionGenerator.getRandomConstant(r)); } else { - sb.append(MariaDBVisitor.asString(expressionGenerator.getRandomExpression())); + expression = MariaDBVisitor.asString(expressionGenerator.getRandomExpression()); } + appendReturningClause(expression); } if (sb.toString().contains("RLIKE") || sb.toString().contains("REGEXP")) { diff --git a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java index 2aceddcc9..a5483ee1d 100644 --- a/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeDeleteGenerator.java @@ -26,12 +26,9 @@ public void buildStatement() { errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - sb.append("DELETE FROM"); - sb.append(" "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, + appendWhereClause(MaterializeVisitor.asString(MaterializeExpressionGenerator.generateExpression(globalState, table.getColumns(), MaterializeDataType.BOOLEAN))); } MaterializeCommon.addCommonExpressionErrors(errors); diff --git a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java index 7d338c027..abd288f99 100644 --- a/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java +++ b/src/sqlancer/materialize/gen/MaterializeUpdateGenerator.java @@ -52,10 +52,9 @@ public void buildStatement() { errors.add("but expression is of type"); MaterializeCommon.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); MaterializeExpression where = MaterializeExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), MaterializeDataType.BOOLEAN); - sb.append(MaterializeVisitor.asString(where)); + appendWhereClause(MaterializeVisitor.asString(where)); } } diff --git a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java index 048ef335f..551264c4a 100644 --- a/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLDeleteGenerator.java @@ -40,8 +40,7 @@ public void buildStatement() { sb.append(" FROM "); sb.append(randomTable.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(MySQLVisitor.asString(gen.generateExpression())); + appendWhereClause(MySQLVisitor.asString(gen.generateExpression())); MySQLErrors.addExpressionErrors(errors); } errors.addAll(Arrays.asList("doesn't have this option", diff --git a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java index 1aca84580..ad13a148a 100644 --- a/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLUpdateGenerator.java @@ -34,9 +34,8 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); MySQLErrors.addExpressionErrors(errors); - sb.append(MySQLVisitor.asString(gen.generateExpression())); + appendWhereClause(MySQLVisitor.asString(gen.generateExpression())); } MySQLErrors.addInsertUpdateErrors(errors); errors.add("doesn't have this option"); diff --git a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java index c46ace304..ec1aa7fc0 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseDeleteGenerator.java @@ -36,8 +36,7 @@ public void buildStatement() { sb.append(" FROM "); sb.append(randomTable.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + appendWhereClause(OceanBaseVisitor.asString(gen.generateExpression())); OceanBaseErrors.addExpressionErrors(errors); } errors.addAll(Arrays.asList("doesn't have this option", "Truncated incorrect DOUBLE value", diff --git a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java index 44fdf39fe..51816e691 100644 --- a/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java +++ b/src/sqlancer/oceanbase/gen/OceanBaseUpdateGenerator.java @@ -39,9 +39,8 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); OceanBaseErrors.addExpressionErrors(errors); - sb.append(OceanBaseVisitor.asString(gen.generateExpression())); + appendWhereClause(OceanBaseVisitor.asString(gen.generateExpression())); errors.add("Data Too Long"); } errors.add("Duplicated primary key"); diff --git a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java index 452f5cdf4..3250b86c8 100644 --- a/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresDeleteGenerator.java @@ -26,20 +26,13 @@ public void buildStatement() { errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - sb.append("DELETE FROM"); + appendDeleteFromTable(table.getName(), Randomly.getBoolean()); if (Randomly.getBoolean()) { - sb.append(" ONLY"); - } - sb.append(" "); - sb.append(table.getName()); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(PostgresVisitor.asString(PostgresExpressionGenerator.generateExpression(globalState, + appendWhereClause(PostgresVisitor.asString(PostgresExpressionGenerator.generateExpression(globalState, table.getColumns(), PostgresDataType.BOOLEAN))); } if (Randomly.getBoolean()) { - sb.append(" RETURNING "); - sb.append(PostgresVisitor + appendReturningClause(PostgresVisitor .asString(PostgresExpressionGenerator.generateExpression(globalState, table.getColumns()))); } PostgresCommon.addCommonExpressionErrors(errors); diff --git a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java index bf70c32d4..c852684a9 100644 --- a/src/sqlancer/postgres/gen/PostgresIndexGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresIndexGenerator.java @@ -107,10 +107,9 @@ public void buildStatement() { sb.append(")"); } if (Randomly.getBoolean()) { - sb.append(" WHERE "); PostgresExpression expr = new PostgresExpressionGenerator(globalState).setColumns(randomTable.getColumns()) .setGlobalState(globalState).generateExpression(PostgresDataType.BOOLEAN); - sb.append(PostgresVisitor.asString(expr)); + appendWhereClause(PostgresVisitor.asString(expr)); } errors.add("already contains data"); // CONCURRENT INDEX failed errors.add("You might need to add explicit type casts"); diff --git a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java index 1effad582..92257a95a 100644 --- a/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresUpdateGenerator.java @@ -52,10 +52,9 @@ public void buildStatement() { errors.add("but expression is of type"); PostgresCommon.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); PostgresExpression where = PostgresExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), PostgresDataType.BOOLEAN); - sb.append(PostgresVisitor.asString(where)); + appendWhereClause(PostgresVisitor.asString(where)); } } diff --git a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java index 59b7b4174..4d5724992 100644 --- a/src/sqlancer/presto/gen/PrestoDeleteGenerator.java +++ b/src/sqlancer/presto/gen/PrestoDeleteGenerator.java @@ -24,12 +24,10 @@ public static SQLQueryAdapter generate(PrestoGlobalState globalState) { @Override public void buildStatement() { - sb.append("DELETE FROM "); PrestoTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(PrestoToStringVisitor + appendWhereClause(PrestoToStringVisitor .asString(new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()))); } diff --git a/src/sqlancer/presto/gen/PrestoIndexGenerator.java b/src/sqlancer/presto/gen/PrestoIndexGenerator.java index 5ec23c773..2d7b75cf8 100644 --- a/src/sqlancer/presto/gen/PrestoIndexGenerator.java +++ b/src/sqlancer/presto/gen/PrestoIndexGenerator.java @@ -51,10 +51,9 @@ public void buildStatement() { } sb.append(")"); if (Randomly.getBoolean()) { - sb.append(" WHERE "); PrestoExpression expr = new PrestoTypedExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(PrestoSchema.PrestoCompositeDataType.getRandomWithoutNull()); - sb.append(PrestoToStringVisitor.asString(expr)); + appendWhereClause(PrestoToStringVisitor.asString(expr)); } errors.add("already exists!"); } diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java index 51b23eaa4..a19da484f 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3DeleteGenerator.java @@ -33,11 +33,9 @@ public static SQLQueryAdapter deleteContent(SQLite3GlobalState globalState, SQLi @Override public void buildStatement() { - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(SQLite3Visitor.asString( + appendWhereClause(SQLite3Visitor.asString( new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } SQLite3Errors.addExpectedExpressionErrors(errors); diff --git a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java index 05f4b9a2e..9230f47ba 100644 --- a/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java +++ b/src/sqlancer/sqlite3/gen/dml/SQLite3UpdateGenerator.java @@ -76,10 +76,9 @@ public void buildStatement() { } if (Randomly.getBoolean()) { - sb.append(" WHERE "); String whereClause = SQLite3Visitor.asString( new SQLite3ExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression()); - sb.append(whereClause); + appendWhereClause(whereClause); } // ORDER BY and LIMIT are only supported by enabling a compile-time option diff --git a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java index cbe5fc721..c83d1a6eb 100644 --- a/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBDeleteGenerator.java @@ -41,8 +41,7 @@ public void buildStatement() { sb.append("FROM "); sb.append(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(TiDBVisitor.asString(gen.generateExpression())); + appendWhereClause(TiDBVisitor.asString(gen.generateExpression())); errors.add("Truncated incorrect"); errors.add("Data truncation"); errors.add("Truncated incorrect FLOAT value"); @@ -54,8 +53,7 @@ public void buildStatement() { .collect(Collectors.joining(", "))); } if (Randomly.getBoolean()) { - sb.append(" LIMIT "); - sb.append(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); + appendLimitClause(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)); } errors.add("Bad Number"); errors.add("Truncated incorrect"); // https://github.com/pingcap/tidb/issues/24292 diff --git a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java index af6430b48..dd79670d9 100644 --- a/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java +++ b/src/sqlancer/tidb/gen/TiDBUpdateGenerator.java @@ -35,9 +35,8 @@ public void buildStatement() { sb.append(" SET "); updateColumns(columns); if (Randomly.getBoolean()) { - sb.append(" WHERE "); TiDBErrors.addExpressionErrors(errors); - sb.append(TiDBVisitor.asString(gen.generateExpression())); + appendWhereClause(TiDBVisitor.asString(gen.generateExpression())); } TiDBErrors.addInsertErrors(errors); } diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java index 6ebe0db5f..af99ae4a5 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLDeleteGenerator.java @@ -23,11 +23,9 @@ public static SQLQueryAdapter generate(YCQLGlobalState globalState) { @Override public void buildStatement() { YCQLTable table = globalState.getSchema().getRandomTable(t -> !t.isView()); - sb.append("DELETE FROM "); - sb.append(table.getName()); + appendDeleteFromTable(table.getName()); if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(YCQLToStringVisitor.asString( + appendWhereClause(YCQLToStringVisitor.asString( new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()).generateExpression())); } YCQLErrors.addExpressionErrors(errors); diff --git a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java index 66c59a968..dd93b38d0 100644 --- a/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ycql/gen/YCQLIndexGenerator.java @@ -35,10 +35,9 @@ public void buildStatement() { sb.append(table.getName()); appendIndexColumnList(table.getRandomNonEmptyColumnSubset(), false); if (Randomly.getBoolean()) { - sb.append(" WHERE "); YCQLExpression expr = new YCQLExpressionGenerator(globalState).setColumns(table.getColumns()) .generateExpression(); - sb.append(YCQLToStringVisitor.asString(expr)); + appendWhereClause(YCQLToStringVisitor.asString(expr)); } errors.add("Query timed out after PT2S"); errors.add("Invalid SQL Statement"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java index 6e35c6862..9d3c7427d 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLDeleteGenerator.java @@ -27,20 +27,13 @@ public void buildStatement() { errors.add("violates foreign key constraint"); errors.add("violates not-null constraint"); errors.add("could not determine which collation to use for string comparison"); - sb.append("DELETE FROM"); + appendDeleteFromTable(table.getName(), Randomly.getBoolean()); if (Randomly.getBoolean()) { - sb.append(" ONLY"); - } - sb.append(" "); - sb.append(table.getName()); - if (Randomly.getBoolean()) { - sb.append(" WHERE "); - sb.append(YSQLVisitor.asString( + appendWhereClause(YSQLVisitor.asString( YSQLExpressionGenerator.generateExpression(globalState, table.getColumns(), YSQLDataType.BOOLEAN))); } if (Randomly.getBoolean()) { - sb.append(" RETURNING "); - sb.append( + appendReturningClause( YSQLVisitor.asString(YSQLExpressionGenerator.generateExpression(globalState, table.getColumns()))); } YSQLErrors.addCommonExpressionErrors(errors); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java index ac5b3242f..5453746b2 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLIndexGenerator.java @@ -93,10 +93,9 @@ public void buildStatement() { sb.append(")"); } if (Randomly.getBoolean()) { - sb.append(" WHERE "); YSQLExpression expr = new YSQLExpressionGenerator(globalState).setColumns(randomTable.getColumns()) .setGlobalState(globalState).generateExpression(YSQLDataType.BOOLEAN); - sb.append(YSQLVisitor.asString(expr)); + appendWhereClause(YSQLVisitor.asString(expr)); } errors.add("already contains data"); // CONCURRENT INDEX failed errors.add("You might need to add explicit type casts"); diff --git a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java index 6c5fd4144..1a9e1de76 100644 --- a/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java +++ b/src/sqlancer/yugabyte/ysql/gen/YSQLUpdateGenerator.java @@ -54,10 +54,9 @@ public void buildStatement() { errors.add("but expression is of type"); YSQLErrors.addCommonExpressionErrors(errors); if (!Randomly.getBooleanWithSmallProbability()) { - sb.append(" WHERE "); YSQLExpression where = YSQLExpressionGenerator.generateExpression(globalState, randomTable.getColumns(), YSQLDataType.BOOLEAN); - sb.append(YSQLVisitor.asString(where)); + appendWhereClause(YSQLVisitor.asString(where)); } } From 1d6722fe4c14db6cdb92b88c29a2486008a96079 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 10:13:07 +0800 Subject: [PATCH 795/856] Remove CnosDB support CnosDB returns EAGAIN ("Tskv: Index: index storage error: Resource temporarily unavailable (os error 11)") on DROP DATABASE under SQLancer's DDL load (cnosdb/cnosdb#2435). The CI job has been red on main for 14+ months, the only published image tags are rolling daily builds (no LTS), and upstream development appears stalled. Pin and retry attempts (#1341) did not help. Remove the CnosDB provider, tests, CI job, and documentation entries. Move CnosDB to the "Previously Supported DBMS" table in CONTRIBUTING.md. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/main.yml | 24 - CONTRIBUTING.md | 2 +- README.md | 2 +- src/check_names.py | 1 - src/sqlancer/Main.java | 2 - src/sqlancer/cnosdb/CnosDBBugs.java | 13 - .../cnosdb/CnosDBComparatorHelper.java | 145 ----- .../cnosdb/CnosDBCompoundDataType.java | 20 - src/sqlancer/cnosdb/CnosDBExpectedError.java | 87 --- src/sqlancer/cnosdb/CnosDBGlobalState.java | 28 - .../cnosdb/CnosDBLoggableFactory.java | 55 -- src/sqlancer/cnosdb/CnosDBOptions.java | 28 - src/sqlancer/cnosdb/CnosDBOracleFactory.java | 39 -- src/sqlancer/cnosdb/CnosDBProvider.java | 123 ----- src/sqlancer/cnosdb/CnosDBSchema.java | 243 -------- .../cnosdb/CnosDBToStringVisitor.java | 278 ---------- src/sqlancer/cnosdb/CnosDBVisitor.java | 102 ---- src/sqlancer/cnosdb/ast/CnosDBAggregate.java | 113 ---- src/sqlancer/cnosdb/ast/CnosDBAlias.java | 35 -- .../cnosdb/ast/CnosDBBetweenOperation.java | 34 -- .../ast/CnosDBBinaryArithmeticOperation.java | 69 --- .../ast/CnosDBBinaryComparisonOperation.java | 57 -- .../ast/CnosDBBinaryLogicalOperation.java | 33 -- .../cnosdb/ast/CnosDBCastOperation.java | 60 -- .../cnosdb/ast/CnosDBColumnValue.java | 27 - .../cnosdb/ast/CnosDBConcatOperation.java | 22 - src/sqlancer/cnosdb/ast/CnosDBConstant.java | 520 ------------------ src/sqlancer/cnosdb/ast/CnosDBExpression.java | 14 - src/sqlancer/cnosdb/ast/CnosDBFunction.java | 30 - .../ast/CnosDBFunctionWithUnknownResult.java | 104 ---- .../cnosdb/ast/CnosDBInOperation.java | 35 -- src/sqlancer/cnosdb/ast/CnosDBJoin.java | 46 -- .../cnosdb/ast/CnosDBLikeOperation.java | 22 - .../cnosdb/ast/CnosDBOrderByTerm.java | 37 -- .../cnosdb/ast/CnosDBPostfixOperation.java | 97 ---- .../cnosdb/ast/CnosDBPostfixText.java | 29 - .../cnosdb/ast/CnosDBPrefixOperation.java | 73 --- src/sqlancer/cnosdb/ast/CnosDBSelect.java | 102 ---- src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java | 28 - src/sqlancer/cnosdb/client/CnosDBClient.java | 110 ---- .../cnosdb/client/CnosDBConnection.java | 27 - .../cnosdb/client/CnosDBException.java | 9 - .../cnosdb/client/CnosDBResultSet.java | 52 -- src/sqlancer/cnosdb/gen/CnosDBCommon.java | 31 -- .../cnosdb/gen/CnosDBExpressionGenerator.java | 461 ---------------- .../cnosdb/gen/CnosDBInsertGenerator.java | 59 -- .../cnosdb/gen/CnosDBTableGenerator.java | 77 --- .../cnosdb/oracle/CnosDBNoRECBase.java | 23 - .../cnosdb/oracle/CnosDBNoRECOracle.java | 171 ------ .../oracle/tlp/CnosDBTLPAggregateOracle.java | 176 ------ .../cnosdb/oracle/tlp/CnosDBTLPBase.java | 112 ---- .../oracle/tlp/CnosDBTLPHavingOracle.java | 65 --- .../oracle/tlp/CnosDBTLPWhereOracle.java | 46 -- .../cnosdb/query/CnosDBOtherQuery.java | 32 -- .../cnosdb/query/CnosDBQueryAdapter.java | 42 -- .../cnosdb/query/CnosDBQueryProvider.java | 6 - .../cnosdb/query/CnosDBSelectQuery.java | 39 -- test/sqlancer/dbms/TestCnosDBNoREC.java | 22 - test/sqlancer/dbms/TestCnosDBTLP.java | 22 - test/sqlancer/dbms/TestConfig.java | 1 - 60 files changed, 2 insertions(+), 4360 deletions(-) delete mode 100644 src/sqlancer/cnosdb/CnosDBBugs.java delete mode 100644 src/sqlancer/cnosdb/CnosDBComparatorHelper.java delete mode 100644 src/sqlancer/cnosdb/CnosDBCompoundDataType.java delete mode 100644 src/sqlancer/cnosdb/CnosDBExpectedError.java delete mode 100644 src/sqlancer/cnosdb/CnosDBGlobalState.java delete mode 100644 src/sqlancer/cnosdb/CnosDBLoggableFactory.java delete mode 100644 src/sqlancer/cnosdb/CnosDBOptions.java delete mode 100644 src/sqlancer/cnosdb/CnosDBOracleFactory.java delete mode 100644 src/sqlancer/cnosdb/CnosDBProvider.java delete mode 100644 src/sqlancer/cnosdb/CnosDBSchema.java delete mode 100644 src/sqlancer/cnosdb/CnosDBToStringVisitor.java delete mode 100644 src/sqlancer/cnosdb/CnosDBVisitor.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBAggregate.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBAlias.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBCastOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBColumnValue.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBConstant.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBExpression.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunction.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBInOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBJoin.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBPostfixText.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBSelect.java delete mode 100644 src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBClient.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBConnection.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBException.java delete mode 100644 src/sqlancer/cnosdb/client/CnosDBResultSet.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBCommon.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java delete mode 100644 src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java delete mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java delete mode 100644 src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java delete mode 100644 src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBOtherQuery.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBQueryProvider.java delete mode 100644 src/sqlancer/cnosdb/query/CnosDBSelectQuery.java delete mode 100644 test/sqlancer/dbms/TestCnosDBNoREC.java delete mode 100644 test/sqlancer/dbms/TestCnosDBTLP.java diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e85c5c219..f0e1b429e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,30 +90,6 @@ jobs: - name: Run Tests run: CITUS_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCitus test - cnosdb: - name: DBMS Tests (CnosDB, creation only) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - distribution: 'temurin' - java-version: '11' - cache: 'maven' - - name: Build SQLancer - run: mvn -B package -DskipTests=true - - name: Set up CnosDB - run: | - docker pull cnosdb/cnosdb:community-latest - docker run --name cnosdb -p 8902:8902 -d cnosdb/cnosdb:community-latest - until nc -z 127.0.0.1 8902 2>/dev/null; do sleep 1; done - - name: Run Tests - run: | - CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBNoREC test - sleep 20 - CNOSDB_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestCnosDBTLP test - clickhouse: name: DBMS Tests (ClickHouse) runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ea5baea1c..76b8833ea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,7 +66,6 @@ Since SQL dialects differ widely, each DBMS to be tested requires a separate imp | YugabyteDB | Working | Typed (YSQL), Untyped (YCQL) | YSQL implementation based on Postgres code. YCQL implementation is primitive for now and uses Cassandra JDBC driver as a proxy interface. | | Databend | Working | Typed | | | QuestDB | Working | Untyped, Generic | The implementation of QuestDB is still WIP, current version covers very basic data types, operations and SQL keywords. | -| CnosDB | Working | Typed | The implementation of CnosDB currently uses Restful API. | | Materialize | Working | Typed | | | Apache Doris | Preliminary | Typed | This is a preliminary implementation, which only contains the common logic of Doris. We have found some errors through it, and hope to improve it in the future. | | Presto | Preliminary | Typed | This is a preliminary implementation, only basic types supported. | @@ -82,6 +81,7 @@ Some DBMS were once supported but subsequently removed. | Cosmos | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because Cosmos is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | | MongoDB | [#915](https://github.com/sqlancer/sqlancer/pull/915) | This implementation was removed because MongoDB is a NoSQL DBMS, while the majority were SQL DBMSs, which resulted in difficulty refactoring SQLancer. | | StoneDB | [#963](https://github.com/sqlancer/sqlancer/pull/963) | This implementation was removed because development of StoneDB stopped. +| CnosDB | | This implementation was removed because the CnosDB image is unstable under SQLancer's DDL load (see [cnosdb/cnosdb#2435](https://github.com/cnosdb/cnosdb/issues/2435)) and the project appears no longer maintained. | ### Unfixed Bugs diff --git a/README.md b/README.md index 134f47666..f41e32d3c 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ Usage: SQLancer [options] [command] [command options] **Understanding SQL generation.** To analyze bug-inducing statements, it is helpful to understand the characteristics of SQLancer. First, SQLancer is expected to always generate SQL statements that are syntactically valid for the DBMS under test. Thus, you should never observe any syntax errors. Second, SQLancer might generate statements that are semantically invalid. For example, SQLancer might attempt to insert duplicate values into a column with a `UNIQUE` constraint, as completely avoiding such semantic errors is challenging. Third, any bug reported by SQLancer is expected to be a real bug, except those reported by CERT (as performance issues are not as clearly defined as other kinds of bugs). If you observe any bugs indicated by SQLancer that you do not consider bugs, something is likely wrong with your setup. Finally, related to the aforementioned point, SQLancer is specific to a version of the DBMS, and you can find the version against which we are tested in our [GitHub Actions workflow](https://github.com/sqlancer/sqlancer/blob/documentation/.github/workflows/main.yml). If you are testing against another version, you might observe various false alarms (e.g., caused by syntax errors). While we would always like for SQLancer to be up-to-date with the latest development version of each DBMS, we lack the resources to achieve this. -**Supported DBMSs.** SQLancer requires DBMS-specific code for each DBMS that it supports. As of January 2025, it provides support for Citus, ClickHouse, CnosDB, CockroachDB, Databend, (Apache) DataFusion, (Apache) Doris, DuckDB, H2, HSQLDB, MariaDB, Materialize, MySQL, OceanBase, PostgreSQL, Presto, QuestDB, SQLite3, TiDB, and YugabyteDB. The extent to which the individual DBMSs are supported [differs](https://github.com/sqlancer/sqlancer/blob/documentation-approaches/CONTRIBUTING.md). +**Supported DBMSs.** SQLancer requires DBMS-specific code for each DBMS that it supports. As of January 2025, it provides support for Citus, ClickHouse, CockroachDB, Databend, (Apache) DataFusion, (Apache) Doris, DuckDB, H2, HSQLDB, MariaDB, Materialize, MySQL, OceanBase, PostgreSQL, Presto, QuestDB, SQLite3, TiDB, and YugabyteDB. The extent to which the individual DBMSs are supported [differs](https://github.com/sqlancer/sqlancer/blob/documentation-approaches/CONTRIBUTING.md). # Approaches and Papers diff --git a/src/check_names.py b/src/check_names.py index f2ab346c6..453580f88 100644 --- a/src/check_names.py +++ b/src/check_names.py @@ -35,7 +35,6 @@ def verify_all_dbs(name_to_files: dict[str:List[str]]): name_to_files: dict[str:List[str]] = dict() name_to_files["Citus"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "citus")) name_to_files["ClickHouse"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "clickhouse")) - name_to_files["CnosDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cnosdb")) name_to_files["CockroachDB"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "cockroachdb")) name_to_files["Databend"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "databend")) name_to_files["DataFusion"] = get_java_files(os.path.join(cwd, "src", "sqlancer", "datafusion")) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index faf35e3c9..f273f5b95 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -26,7 +26,6 @@ import sqlancer.citus.CitusProvider; import sqlancer.clickhouse.ClickHouseProvider; -import sqlancer.cnosdb.CnosDBProvider; import sqlancer.cockroachdb.CockroachDBProvider; import sqlancer.common.log.Loggable; import sqlancer.common.query.Query; @@ -750,7 +749,6 @@ private static void checkForIssue799(List> providers) "No DBMS implementations (i.e., instantiations of the DatabaseProvider class) were found. You likely ran into an issue described in https://github.com/sqlancer/sqlancer/issues/799. As a workaround, I now statically load all supported providers as of June 7, 2023."); providers.add(new CitusProvider()); providers.add(new ClickHouseProvider()); - providers.add(new CnosDBProvider()); providers.add(new CockroachDBProvider()); providers.add(new DatabendProvider()); providers.add(new DorisProvider()); diff --git a/src/sqlancer/cnosdb/CnosDBBugs.java b/src/sqlancer/cnosdb/CnosDBBugs.java deleted file mode 100644 index 4e6eb96e9..000000000 --- a/src/sqlancer/cnosdb/CnosDBBugs.java +++ /dev/null @@ -1,13 +0,0 @@ -package sqlancer.cnosdb; - -public final class CnosDBBugs { - - // https://github.com/cnosdb/cnosdb/issues/786 - public static final boolean BUG786 = true; - - // https://github.com/apache/arrow-rs/issues/3547 - public static final boolean BUG3547 = true; - - private CnosDBBugs() { - } -} diff --git a/src/sqlancer/cnosdb/CnosDBComparatorHelper.java b/src/sqlancer/cnosdb/CnosDBComparatorHelper.java deleted file mode 100644 index 46b6ba615..000000000 --- a/src/sqlancer/cnosdb/CnosDBComparatorHelper.java +++ /dev/null @@ -1,145 +0,0 @@ -package sqlancer.cnosdb; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.function.UnaryOperator; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.cnosdb.query.CnosDBSelectQuery; -import sqlancer.common.query.ExpectedErrors; - -public final class CnosDBComparatorHelper { - - private CnosDBComparatorHelper() { - } - - public static List getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors, - CnosDBGlobalState state) throws Exception { - if (state.getOptions().logEachSelect()) { - // TODO: refactor me - state.getLogger().writeCurrent(queryString); - try { - state.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, errors); - List result = new ArrayList<>(); - CnosDBResultSet resultSet; - try { - q.executeAndGet(state); - resultSet = q.getResultSet(); - if (resultSet == null) { - throw new AssertionError(q); - } - while (resultSet.next()) { - result.add(resultSet.getString(1)); - } - } catch (Exception e) { - if (e instanceof IgnoreMeException) { - throw e; - } - if (e instanceof NumberFormatException) { - throw new IgnoreMeException(); - } - if (e.getMessage() == null) { - throw new AssertionError(queryString, e); - } - if (errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(queryString, e); - } - - return result; - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - String originalQueryString, List combinedString, CnosDBGlobalState state) { - if (resultSet.size() != secondResultSet.size()) { - String queryFormatString = "-- %s;\n-- cardinality: %d"; - String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size()); - String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), - secondResultSet.size()); - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s", - resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString); - throw new AssertionError(assertionMessage); - } - - Set firstHashSet = new HashSet<>(resultSet); - Set secondHashSet = new HashSet<>(secondResultSet); - - if (!firstHashSet.equals(secondHashSet)) { - Set firstResultSetMisses = new HashSet<>(firstHashSet); - firstResultSetMisses.removeAll(secondHashSet); - Set secondResultSetMisses = new HashSet<>(secondHashSet); - secondResultSetMisses.removeAll(firstHashSet); - String queryFormatString = "-- %s;\n-- misses: %s"; - String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses); - String secondQueryString = String.format(queryFormatString, String.join(";", combinedString), - secondResultSetMisses); - // update the SELECT queries to be logged at the bottom of the error log file - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s", - firstQueryString, secondQueryString); - throw new AssertionError(assertionMessage); - } - } - - public static void assumeResultSetsAreEqual(List resultSet, List secondResultSet, - String originalQueryString, List combinedString, CnosDBGlobalState state, - UnaryOperator canonicalizationRule) { - // Overloaded version of assumeResultSetsAreEqual that takes a canonicalization function which is applied to - // both result sets before their comparison. - List canonicalizedResultSet = resultSet.stream().map(canonicalizationRule).collect(Collectors.toList()); - List canonicalizedSecondResultSet = secondResultSet.stream().map(canonicalizationRule) - .collect(Collectors.toList()); - assumeResultSetsAreEqual(canonicalizedResultSet, canonicalizedSecondResultSet, originalQueryString, - combinedString, state); - } - - public static List getCombinedResultSet(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, - ExpectedErrors errors) throws Exception { - List secondResultSet; - if (asUnion) { - String unionString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL " - + thirdQueryString; - combinedString.add(unionString); - secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); - } else { - secondResultSet = new ArrayList<>(); - secondResultSet.addAll(getResultSetFirstColumnAsString(firstQueryString, errors, state)); - secondResultSet.addAll(getResultSetFirstColumnAsString(secondQueryString, errors, state)); - secondResultSet.addAll(getResultSetFirstColumnAsString(thirdQueryString, errors, state)); - combinedString.add(firstQueryString); - combinedString.add(secondQueryString); - combinedString.add(thirdQueryString); - } - return secondResultSet; - } - - public static List getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString, - String thirdQueryString, List combinedString, boolean asUnion, CnosDBGlobalState state, - ExpectedErrors errors) throws Exception { - String unionString; - if (asUnion) { - unionString = firstQueryString + " UNION " + secondQueryString + " UNION " + thirdQueryString; - } else { - unionString = "SELECT DISTINCT * FROM (" + firstQueryString + " UNION ALL " + secondQueryString - + " UNION ALL " + thirdQueryString + ")"; - } - List secondResultSet; - combinedString.add(unionString); - secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state); - return secondResultSet; - } -} diff --git a/src/sqlancer/cnosdb/CnosDBCompoundDataType.java b/src/sqlancer/cnosdb/CnosDBCompoundDataType.java deleted file mode 100644 index 034f0fc90..000000000 --- a/src/sqlancer/cnosdb/CnosDBCompoundDataType.java +++ /dev/null @@ -1,20 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public final class CnosDBCompoundDataType { - - private final CnosDBDataType dataType; - - private CnosDBCompoundDataType(CnosDBDataType dataType) { - this.dataType = dataType; - } - - public static CnosDBCompoundDataType create(CnosDBDataType type) { - return new CnosDBCompoundDataType(type); - } - - public CnosDBDataType getDataType() { - return dataType; - } -} diff --git a/src/sqlancer/cnosdb/CnosDBExpectedError.java b/src/sqlancer/cnosdb/CnosDBExpectedError.java deleted file mode 100644 index 61dba101b..000000000 --- a/src/sqlancer/cnosdb/CnosDBExpectedError.java +++ /dev/null @@ -1,87 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.common.query.ExpectedErrors; - -public final class CnosDBExpectedError { - - private CnosDBExpectedError() { - } - - public static List getExpectedErrors() { - ArrayList errors = new ArrayList<>(); - - errors.add("have the same name. Consider aliasing"); - errors.add( - "error: Optimizer rule 'projection_push_down' failed due to unexpected error: Schema error: Schema contains duplicate qualified field name"); - errors.add("Projection references non-aggregate values:"); - errors.add("External err: Schema error: No field named"); - errors.add( - "Optimizer rule 'common_sub_expression_eliminate' failed due to unexpected error: Schema error: No field named"); - errors.add("Binary"); - errors.add("Invalid pattern in LIKE expression"); - errors.add("If the projection contains the time column, it must contain the field column."); - errors.add("Schema error: No field named"); - errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:"); - errors.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error"); - errors.add("Schema error: No field named "); - errors.add("err: External err: Schema error: No field named"); - errors.add("Optimizer rule 'simplify_expressions' failed due to unexpected error"); - errors.add("Csv error: CSV Writer does not support List"); - errors.add("This feature is not implemented: cross join."); - errors.add("Execution error: field position must be greater than zero"); - errors.add("First argument of `DATE_PART` must be non-null scalar Utf8"); - errors.add("Cannot create filter with non-boolean predicate 'NULL' returning Null"); - errors.add("requested character too large for encoding."); - errors.add("Can not find compatible types to compare Boolean with [Utf8]."); - errors.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT"); - errors.add("HAVING clause references non-aggregate values:"); - errors.add("Cannot create filter with non-boolean predicate"); - errors.add("negative substring length not allowed"); - errors.add("The function Sum does not support inputs of type Boolean."); - errors.add("The function Avg does not support inputs of type Boolean."); - errors.add("Percentile value must be between 0.0 and 1.0 inclusive"); - errors.add("Date part '' not supported"); - errors.add("Min/Max accumulator not implemented for type Boolean."); - errors.add("meta need get_series_id_by_filter"); - errors.add("Arrow: Cast error:"); - errors.add("Arrow error: Cast error:"); - errors.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:"); - errors.add("Arrow error: Divide by zero error"); - errors.add("desired percentile argument must be float literal"); - errors.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)"); - errors.add("Execution error: Date part"); - errors.add("Physical plan does not support logical expression MIN(Boolean"); - errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64"); - errors.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64."); - errors.add("Data type UInt64 not supported for binary operation '#' on dyn arrays."); - errors.add("Arrow: Divide by zero error"); - errors.add("The function ApproxPercentileCont does not support inputs of type Null."); - errors.add("can't be evaluated because there isn't a common type to coerce the types to"); - errors.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug"); - errors.add("The function ApproxMedian does not support inputs of type Null."); - errors.add("null character not permitted."); - errors.add("The percentile argument for ApproxPercentileCont must be Float64, not Null."); - errors.add("This feature is not implemented"); - errors.add("The function Avg does not support inputs of type Null."); - errors.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]"); - errors.add( - "Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed."); - errors.add("Coercion from"); - - errors.add("Error parsing timestamp"); - errors.add("lpad requested length"); - errors.add("rpad requested length"); - errors.add("No function matches the given name and argument types"); - return errors; - } - - public static ExpectedErrors expectedErrors() { - ExpectedErrors res = new ExpectedErrors(); - res.addAll(getExpectedErrors()); - return res; - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBGlobalState.java b/src/sqlancer/cnosdb/CnosDBGlobalState.java deleted file mode 100644 index 9f34e03a5..000000000 --- a/src/sqlancer/cnosdb/CnosDBGlobalState.java +++ /dev/null @@ -1,28 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.ExecutionTimer; -import sqlancer.GlobalState; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.query.Query; - -public class CnosDBGlobalState extends GlobalState { - - @Override - protected void executeEpilogue(Query q, boolean success, ExecutionTimer timer) throws Exception { - boolean logExecutionTime = getOptions().logExecutionTime(); - if (success && getOptions().printSucceedingStatements()) { - System.out.println(q.getQueryString()); - } - if (logExecutionTime) { - getLogger().writeCurrent(" -- " + timer.end().asString()); - } - if (q.couldAffectSchema()) { - updateSchema(); - } - } - - @Override - public CnosDBSchema readSchema() throws Exception { - return CnosDBSchema.fromConnection(getConnection()); - } -} diff --git a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java b/src/sqlancer/cnosdb/CnosDBLoggableFactory.java deleted file mode 100644 index 407621c8b..000000000 --- a/src/sqlancer/cnosdb/CnosDBLoggableFactory.java +++ /dev/null @@ -1,55 +0,0 @@ -package sqlancer.cnosdb; - -import java.io.PrintWriter; -import java.io.StringWriter; - -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.cnosdb.query.CnosDBQueryAdapter; -import sqlancer.common.log.Loggable; -import sqlancer.common.log.LoggableFactory; -import sqlancer.common.log.LoggedString; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; - -public class CnosDBLoggableFactory extends LoggableFactory { - - @Override - protected Loggable createLoggable(String input, String suffix) { - String completeString = input; - if (!input.endsWith(";")) { - completeString += ";"; - } - if (suffix != null && !suffix.isEmpty()) { - completeString += suffix; - } - return new LoggedString(completeString); - } - - @Override - public CnosDBQueryAdapter getQueryForStateToReproduce(String queryString) { - return new CnosDBOtherQuery(queryString, CnosDBExpectedError.expectedErrors()); - } - - @Override - public CnosDBQueryAdapter commentOutQuery(Query query) { - String queryString = query.getLogString(); - String newQueryString = "-- " + queryString; - ExpectedErrors errors = new ExpectedErrors(); - return new CnosDBOtherQuery(newQueryString, errors); - } - - @Override - protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) { - String sb = "-- Time: " + time + "\n" + "-- Database: " + databaseName + "\n" + "-- Database version: " - + databaseVersion + "\n" + "-- seed value: " + seedValue + "\n"; - return new LoggedString(sb); - } - - @Override - public Loggable convertStacktraceToLoggable(Throwable throwable) { - StringWriter sw = new StringWriter(); - PrintWriter pw = new PrintWriter(sw); - throwable.printStackTrace(pw); - return new LoggedString("--" + sw.toString().replace("\n", "\n--")); - } -} diff --git a/src/sqlancer/cnosdb/CnosDBOptions.java b/src/sqlancer/cnosdb/CnosDBOptions.java deleted file mode 100644 index f101c2d38..000000000 --- a/src/sqlancer/cnosdb/CnosDBOptions.java +++ /dev/null @@ -1,28 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.List; - -import com.beust.jcommander.Parameter; -import com.beust.jcommander.Parameters; - -import sqlancer.DBMSSpecificOptions; - -@Parameters(separators = "=", commandDescription = "CnosDB (default port: " + CnosDBOptions.DEFAULT_PORT - + ", default host: " + CnosDBOptions.DEFAULT_HOST + ")") -public class CnosDBOptions implements DBMSSpecificOptions { - - public static final String DEFAULT_HOST = "localhost"; - public static final int DEFAULT_PORT = 31001; - - @Parameter(names = "--oracle", description = "Specifies which test oracle should be used for CnosDB") - public List oracle = List.of(CnosDBOracleFactory.QUERY_PARTITIONING); - - @Parameter(names = "--connection-url", description = "Specifies the URL for connecting to the CnosDB", arity = 1) - public String connectionURL = String.format("http://%s:%d", CnosDBOptions.DEFAULT_HOST, CnosDBOptions.DEFAULT_PORT); - - @Override - public List getTestOracleFactory() { - return oracle; - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBOracleFactory.java b/src/sqlancer/cnosdb/CnosDBOracleFactory.java deleted file mode 100644 index 7cb9c4fc6..000000000 --- a/src/sqlancer/cnosdb/CnosDBOracleFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.OracleFactory; -import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPAggregateOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPHavingOracle; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPWhereOracle; -import sqlancer.common.oracle.CompositeTestOracle; -import sqlancer.common.oracle.TestOracle; - -public enum CnosDBOracleFactory implements OracleFactory { - NOREC { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBNoRECOracle(globalState); - } - }, - HAVING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - return new CnosDBTLPHavingOracle(globalState); - } - - }, - QUERY_PARTITIONING { - @Override - public TestOracle create(CnosDBGlobalState globalState) { - List> oracles = new ArrayList<>(); - oracles.add(new CnosDBTLPWhereOracle(globalState)); - oracles.add(new CnosDBTLPHavingOracle(globalState)); - oracles.add(new CnosDBTLPAggregateOracle(globalState)); - return new CompositeTestOracle<>(oracles, globalState); - } - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBProvider.java b/src/sqlancer/cnosdb/CnosDBProvider.java deleted file mode 100644 index 8b69c53b3..000000000 --- a/src/sqlancer/cnosdb/CnosDBProvider.java +++ /dev/null @@ -1,123 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.Objects; - -import com.google.auto.service.AutoService; - -import sqlancer.AbstractAction; -import sqlancer.DatabaseProvider; -import sqlancer.IgnoreMeException; -import sqlancer.ProviderAdapter; -import sqlancer.Randomly; -import sqlancer.StatementExecutor; -import sqlancer.cnosdb.client.CnosDBClient; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.cnosdb.gen.CnosDBInsertGenerator; -import sqlancer.cnosdb.gen.CnosDBTableGenerator; -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.cnosdb.query.CnosDBQueryProvider; -import sqlancer.common.log.LoggableFactory; - -@AutoService(DatabaseProvider.class) -public class CnosDBProvider extends ProviderAdapter { - - protected String username; - protected String password; - protected String host; - protected int port; - protected String databaseName; - - public CnosDBProvider() { - super(CnosDBGlobalState.class, CnosDBOptions.class); - } - - protected CnosDBProvider(Class globalClass, Class optionClass) { - super(globalClass, optionClass); - } - - protected static int mapActions(CnosDBGlobalState globalState, Action a) { - Randomly r = globalState.getRandomly(); - int nrPerformed; - if (Objects.requireNonNull(a) == Action.INSERT) { - nrPerformed = r.getInteger(0, globalState.getOptions().getMaxNumberInserts()); - } else { - throw new AssertionError(a); - } - return nrPerformed; - - } - - @Override - protected void checkViewsAreValid(CnosDBGlobalState globalState) { - } - - @Override - public void generateDatabase(CnosDBGlobalState globalState) throws Exception { - createTables(globalState, Randomly.fromOptions(4, 5, 6)); - prepareTables(globalState); - - } - - @Override - public CnosDBConnection createDatabase(CnosDBGlobalState globalState) throws Exception { - - username = globalState.getOptions().getUserName(); - password = globalState.getOptions().getPassword(); - host = globalState.getOptions().getHost(); - port = globalState.getOptions().getPort(); - databaseName = globalState.getDatabaseName(); - CnosDBClient client = new CnosDBClient(host, port, username, password, databaseName); - CnosDBConnection connection = new CnosDBConnection(client); - client.execute("DROP DATABASE IF EXISTS " + databaseName); - globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName); - client.execute("CREATE DATABASE " + databaseName); - globalState.getState().logStatement("CREATE DATABASE " + databaseName); - - return connection; - } - - protected void createTables(CnosDBGlobalState globalState, int numTables) throws Exception { - while (globalState.getSchema().getDatabaseTables().size() < numTables) { - String tableName = String.format("m%d", globalState.getSchema().getDatabaseTables().size()); - CnosDBOtherQuery createTable = CnosDBTableGenerator.generate(tableName); - globalState.executeStatement(createTable); - } - } - - protected void prepareTables(CnosDBGlobalState globalState) throws Exception { - StatementExecutor se = new StatementExecutor<>(globalState, Action.values(), - CnosDBProvider::mapActions, (q) -> { - if (globalState.getSchema().getDatabaseTables().isEmpty()) { - throw new IgnoreMeException(); - } - }); - se.executeStatements(); - } - - @Override - public String getDBMSName() { - return "CnosDB".toLowerCase(); - } - - @Override - public LoggableFactory getLoggableFactory() { - return new CnosDBLoggableFactory(); - } - - public enum Action implements AbstractAction { - INSERT(CnosDBInsertGenerator::insert); - - private final CnosDBQueryProvider sqlQueryProvider; - - Action(CnosDBQueryProvider sqlQueryProvider) { - this.sqlQueryProvider = sqlQueryProvider; - } - - @Override - public CnosDBOtherQuery getQuery(CnosDBGlobalState state) throws Exception { - return new CnosDBOtherQuery(sqlQueryProvider.getQuery(state).getQueryString(), - CnosDBExpectedError.expectedErrors()); - } - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBSchema.java b/src/sqlancer/cnosdb/CnosDBSchema.java deleted file mode 100644 index 022969ce5..000000000 --- a/src/sqlancer/cnosdb/CnosDBSchema.java +++ /dev/null @@ -1,243 +0,0 @@ -package sqlancer.cnosdb; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import sqlancer.Randomly; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.common.schema.AbstractRowValue; -import sqlancer.common.schema.AbstractSchema; -import sqlancer.common.schema.AbstractTable; -import sqlancer.common.schema.AbstractTableColumn; -import sqlancer.common.schema.AbstractTables; -import sqlancer.common.schema.TableIndex; - -public class CnosDBSchema extends AbstractSchema { - - private final String databaseName; - - public CnosDBSchema(List databaseTables, String databaseName) { - super(databaseTables); - this.databaseName = databaseName; - } - - public static CnosDBDataType getColumnType(String typeString) { - switch (typeString.toLowerCase()) { - case "bigint": - return CnosDBDataType.INT; - case "boolean": - return CnosDBDataType.BOOLEAN; - case "string": - return CnosDBDataType.STRING; - case "double": - return CnosDBDataType.DOUBLE; - case "bigint unsigned": - case "unsigned": - return CnosDBDataType.UINT; - case "timestamp(nanosecond)": - return CnosDBDataType.TIMESTAMP; - default: - throw new AssertionError(typeString); - } - } - - public static CnosDBSchema fromConnection(CnosDBConnection con) throws Exception { - CnosDBResultSet tablesRes = con.getClient().executeQuery("SHOW TABLES"); - - List tables = new ArrayList<>(); - while (tablesRes.next()) { - String tableName = tablesRes.getString(1); - List columns = getTableColumns(con, tableName); - tables.add(new CnosDBTable(tableName, columns)); - } - - return new CnosDBSchema(tables, con.getClient().getDatabase()); - } - - protected static List getTableColumns(CnosDBConnection con, String tableName) throws Exception { - CnosDBResultSet columnsRes = con.getClient().executeQuery("DESCRIBE TABLE " + tableName); - List columns = new ArrayList<>(); - CnosDBTable table = new CnosDBTable(tableName, columns); - while (columnsRes.next()) { - String columnName = columnsRes.getString(1); - String columnType = columnsRes.getString(3).toLowerCase(); - CnosDBDataType dataType = CnosDBSchema.getColumnType(columnsRes.getString(2)); - CnosDBColumn column; - if (columnType.contentEquals("time")) { - column = new CnosDBTimeColumn(); - } else if (columnType.contentEquals("tag")) { - column = new CnosDBTagColumn(columnName); - } else { - column = new CnosDBFieldColumn(columnName, dataType); - } - column.setTable(table); - columns.add(column); - } - - return columns; - } - - public CnosDBTables getRandomTableNonEmptyTables() { - return new CnosDBTables(Randomly.nonEmptySubset(getDatabaseTables())); - } - - public String getDatabaseName() { - return databaseName; - } - - public enum CnosDBDataType { - INT, BOOLEAN, STRING, DOUBLE, UINT, TIMESTAMP; - - public static CnosDBDataType getRandomType() { - return Randomly.fromOptions(values()); - } - - public static CnosDBDataType getRandomTypeWithoutTimeStamp() { - List dataTypes = new ArrayList<>(Arrays.asList(values())); - dataTypes.remove(TIMESTAMP); - return Randomly.fromList(dataTypes); - } - } - - public static class CnosDBColumn extends AbstractTableColumn { - - public CnosDBColumn(String name, CnosDBDataType columnType) { - super(name, null, columnType); - } - - public static CnosDBColumn createDummy(String name) { - return new CnosDBColumn(name, CnosDBDataType.INT); - } - - } - - public static class CnosDBTagColumn extends CnosDBColumn { - public CnosDBTagColumn(String name) { - super(name, CnosDBDataType.STRING); - } - } - - public static class CnosDBTimeColumn extends CnosDBColumn { - public CnosDBTimeColumn() { - super("TIME", CnosDBDataType.TIMESTAMP); - } - } - - public static class CnosDBFieldColumn extends CnosDBColumn { - public CnosDBFieldColumn(String name, CnosDBDataType columnType) { - super(name, columnType); - assert columnType != CnosDBDataType.TIMESTAMP; - } - } - - public static class CnosDBTables extends AbstractTables { - - public CnosDBTables(List tables) { - super(tables); - } - - public CnosDBRowValue getRandomRowValue(CnosDBConnection con) { - return null; - } - - public List getRandomColumnsWithOnlyOneField() { - ArrayList res = new ArrayList<>(); - this.getTables().forEach(table -> res.addAll(table.getRandomColumnsWithOnlyOneField())); - return res; - } - - } - - public static class CnosDBRowValue extends AbstractRowValue { - - protected CnosDBRowValue(CnosDBTables tables, Map values) { - super(tables, values); - } - - } - - public static class CnosDBTable extends AbstractTable { - - public CnosDBTable(String tableName, List columns) { - super(tableName, columns, null, false); - } - - @Override - public List getColumns() { - List res = super.getColumns(); - boolean hasTime = false; - for (CnosDBColumn column : res) { - if (column instanceof CnosDBTimeColumn) { - hasTime = true; - break; - } - } - assert hasTime; - - return res; - } - - public List getRandomColumnsWithOnlyOneField() { - ArrayList res = new ArrayList<>(); - boolean hasField = false; - for (CnosDBColumn column : getColumns()) { - if (column instanceof CnosDBTagColumn && Randomly.getBoolean()) { - res.add(column); - } else if (column instanceof CnosDBFieldColumn && !hasField) { - res.add(column); - hasField = true; - } - } - return res; - } - - // SELECT COUNT(*) FROM table; - @Override - public long getNrRows(CnosDBGlobalState globalState) { - long res; - try { - CnosDBResultSet tableCountRes = globalState.getConnection().getClient() - .executeQuery("SELECT COUNT(time) FROM " + this.name); - tableCountRes.next(); - res = tableCountRes.getLong(1); - } catch (Exception e) { - res = 0; - } - return res; - } - - @Override - public List getRandomNonEmptyColumnSubset() { - List selectedColumns = new ArrayList<>(); - ArrayList remainingColumns = new ArrayList<>(this.getColumns()); - - remainingColumns.removeIf(column -> column instanceof CnosDBTimeColumn); - CnosDBTimeColumn timeColumn = new CnosDBTimeColumn(); - timeColumn.setTable(this); - selectedColumns.add(timeColumn); - - remainingColumns.stream().filter(column -> column instanceof CnosDBTagColumn).findFirst().ifPresent(tag -> { - selectedColumns.add(tag); - remainingColumns.remove(tag); - }); - - remainingColumns.stream().filter(column -> column instanceof CnosDBFieldColumn).findFirst() - .ifPresent(field -> { - selectedColumns.add(field); - remainingColumns.remove(field); - }); - - int nr = Math.min(Randomly.smallNumber() + 1, remainingColumns.size()); - for (int i = 0; i < nr; i++) { - selectedColumns - .add(remainingColumns.remove((int) Randomly.getNotCachedInteger(0, remainingColumns.size()))); - } - return selectedColumns; - } - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java b/src/sqlancer/cnosdb/CnosDBToStringVisitor.java deleted file mode 100644 index 388e2ccd8..000000000 --- a/src/sqlancer/cnosdb/CnosDBToStringVisitor.java +++ /dev/null @@ -1,278 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.Randomly; -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBBetweenOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBFunction; -import sqlancer.cnosdb.ast.CnosDBInOperation; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBLikeOperation; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixText; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.ast.CnosDBSimilarTo; -import sqlancer.common.visitor.BinaryOperation; -import sqlancer.common.visitor.ToStringVisitor; - -public final class CnosDBToStringVisitor extends ToStringVisitor implements CnosDBVisitor { - - @Override - public void visitSpecific(CnosDBExpression expr) { - CnosDBVisitor.super.visit(expr); - } - - @Override - public void visit(CnosDBConstant constant) { - sb.append(constant.getTextRepresentation()); - } - - @Override - public String get() { - return sb.toString(); - } - - @Override - public void visit(CnosDBPostfixOperation op) { - sb.append("("); - visit(op.getExpression()); - sb.append(")"); - sb.append(" "); - sb.append(op.getOperatorTextRepresentation()); - } - - @Override - public void visit(CnosDBColumnValue c) { - sb.append(c.getColumn().getFullQualifiedName()); - } - - @Override - public void visit(CnosDBPrefixOperation op) { - sb.append(op.getTextRepresentation()); - sb.append(" ("); - visit(op.getExpression()); - sb.append(")"); - } - - @Override - public void visit(CnosDBFromTable from) { - sb.append(from.getTable().getName()); - } - - @Override - public void visit(CnosDBSubquery subquery) { - sb.append("("); - visit(subquery.getSelect()); - sb.append(") AS "); - sb.append(subquery.getName()); - } - - @Override - public void visit(CnosDBSelect s) { - sb.append("SELECT "); - switch (s.getSelectOption()) { - case DISTINCT: - sb.append("DISTINCT "); - if (s.getDistinctOnClause() != null) { - sb.append("ON ("); - visit(s.getDistinctOnClause()); - sb.append(") "); - } - break; - case ALL: - sb.append(Randomly.fromOptions("ALL ", "")); - break; - default: - throw new AssertionError(); - } - if (s.getFetchColumns() == null) { - sb.append("*"); - } else { - visit(s.getFetchColumns()); - } - sb.append(" FROM "); - visit(s.getFromList()); - - for (CnosDBJoin j : s.getJoinClauses()) { - sb.append(" "); - switch (j.getType()) { - case INNER: - if (Randomly.getBoolean()) { - sb.append("INNER "); - } - sb.append("JOIN"); - break; - case LEFT: - sb.append("LEFT OUTER JOIN"); - break; - case RIGHT: - sb.append("RIGHT OUTER JOIN"); - break; - case FULL: - sb.append("FULL OUTER JOIN"); - break; - // case CROSS: - // sb.append("CROSS JOIN"); - // break; - default: - throw new AssertionError(j.getType()); - } - sb.append(" "); - visit(j.getTableReference()); - // if (j.getType() != CnosDBJoinType.CROSS) { - sb.append(" ON "); - visit(j.getOnClause()); - // } - } - - if (s.getWhereClause() != null) { - sb.append(" WHERE "); - visit(s.getWhereClause()); - } - if (!s.getGroupByExpressions().isEmpty()) { - sb.append(" GROUP BY "); - visit(s.getGroupByExpressions()); - } - if (s.getHavingClause() != null) { - sb.append(" HAVING "); - visit(s.getHavingClause()); - - } - if (!s.getOrderByClauses().isEmpty()) { - sb.append(" ORDER BY "); - visit(s.getOrderByClauses()); - } - if (s.getLimitClause() != null) { - sb.append(" LIMIT "); - visit(s.getLimitClause()); - } - - if (s.getOffsetClause() != null) { - sb.append(" OFFSET "); - visit(s.getOffsetClause()); - } - } - - @Override - public void visit(CnosDBOrderByTerm op) { - visit(op.getExpr()); - sb.append(" "); - sb.append(op.getOrder()); - } - - @Override - public void visit(CnosDBFunction f) { - sb.append(f.getFunctionName()); - sb.append("("); - int i = 0; - for (CnosDBExpression arg : f.getArguments()) { - if (i++ != 0) { - sb.append(", "); - } - visit(arg); - } - sb.append(")"); - } - - @Override - public void visit(CnosDBCastOperation cast) { - sb.append("CAST( "); - visit(cast.getExpression()); - sb.append(" AS "); - appendType(cast); - sb.append(")"); - } - - private void appendType(CnosDBCastOperation cast) { - CnosDBCompoundDataType compoundType = cast.getCompoundType(); - switch (compoundType.getDataType()) { - case BOOLEAN: - sb.append("BOOLEAN"); - break; - case INT: - sb.append("BIGINT"); - break; - case STRING: - sb.append(Randomly.fromOptions("STRING")); - break; - case DOUBLE: - sb.append("DOUBLE"); - break; - case UINT: - sb.append("BIGINT UNSIGNED"); - break; - case TIMESTAMP: - sb.append("TIMESTAMP"); - break; - - default: - throw new AssertionError(cast.getType()); - } - } - - @Override - public void visit(CnosDBBetweenOperation op) { - sb.append("("); - visit(op.getExpr()); - sb.append(") BETWEEN ("); - visit(op.getLeft()); - sb.append(") AND ("); - visit(op.getRight()); - sb.append(")"); - } - - @Override - public void visit(CnosDBInOperation op) { - sb.append("("); - visit(op.getExpr()); - sb.append(")"); - if (!op.isTrue()) { - sb.append(" NOT"); - } - sb.append(" IN ("); - visit(op.getListElements()); - sb.append(")"); - } - - @Override - public void visit(CnosDBPostfixText op) { - visit(op.getExpr()); - sb.append(op.getText()); - } - - @Override - public void visit(CnosDBAggregate op) { - sb.append(op.getFunction()); - sb.append("("); - visit(op.getArgs()); - sb.append(")"); - } - - @Override - public void visit(CnosDBSimilarTo op) { - sb.append("("); - visit(op.getString()); - sb.append(" SIMILAR TO "); - visit(op.getSimilarTo()); - sb.append(")"); - } - - @Override - public void visit(CnosDBBinaryLogicalOperation op) { - super.visit((BinaryOperation) op); - } - - @Override - public void visit(CnosDBLikeOperation op) { - super.visit((BinaryOperation) op); - } - -} diff --git a/src/sqlancer/cnosdb/CnosDBVisitor.java b/src/sqlancer/cnosdb/CnosDBVisitor.java deleted file mode 100644 index 7c1af7224..000000000 --- a/src/sqlancer/cnosdb/CnosDBVisitor.java +++ /dev/null @@ -1,102 +0,0 @@ -package sqlancer.cnosdb; - -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBBetweenOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBFunction; -import sqlancer.cnosdb.ast.CnosDBInOperation; -import sqlancer.cnosdb.ast.CnosDBLikeOperation; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixText; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.ast.CnosDBSimilarTo; - -public interface CnosDBVisitor { - - static String asString(CnosDBExpression expr) { - CnosDBToStringVisitor visitor = new CnosDBToStringVisitor(); - visitor.visit(expr); - return visitor.get(); - } - - void visit(CnosDBConstant constant); - - void visit(CnosDBPostfixOperation op); - - void visit(CnosDBColumnValue c); - - void visit(CnosDBPrefixOperation op); - - void visit(CnosDBSelect op); - - void visit(CnosDBOrderByTerm op); - - void visit(CnosDBFunction f); - - void visit(CnosDBCastOperation cast); - - void visit(CnosDBBetweenOperation op); - - void visit(CnosDBInOperation op); - - void visit(CnosDBPostfixText op); - - void visit(CnosDBAggregate op); - - void visit(CnosDBFromTable from); - - void visit(CnosDBSubquery subquery); - - void visit(CnosDBBinaryLogicalOperation op); - - void visit(CnosDBLikeOperation op); - - void visit(CnosDBSimilarTo op); - - default void visit(CnosDBExpression expression) { - if (expression instanceof CnosDBConstant) { - visit((CnosDBConstant) expression); - } else if (expression instanceof CnosDBPostfixOperation) { - visit((CnosDBPostfixOperation) expression); - } else if (expression instanceof CnosDBColumnValue) { - visit((CnosDBColumnValue) expression); - } else if (expression instanceof CnosDBPrefixOperation) { - visit((CnosDBPrefixOperation) expression); - } else if (expression instanceof CnosDBSelect) { - visit((CnosDBSelect) expression); - } else if (expression instanceof CnosDBOrderByTerm) { - visit((CnosDBOrderByTerm) expression); - } else if (expression instanceof CnosDBFunction) { - visit((CnosDBFunction) expression); - } else if (expression instanceof CnosDBCastOperation) { - visit((CnosDBCastOperation) expression); - } else if (expression instanceof CnosDBBetweenOperation) { - visit((CnosDBBetweenOperation) expression); - } else if (expression instanceof CnosDBInOperation) { - visit((CnosDBInOperation) expression); - } else if (expression instanceof CnosDBAggregate) { - visit((CnosDBAggregate) expression); - } else if (expression instanceof CnosDBPostfixText) { - visit((CnosDBPostfixText) expression); - } else if (expression instanceof CnosDBSimilarTo) { - visit((CnosDBSimilarTo) expression); - } else if (expression instanceof CnosDBFromTable) { - visit((CnosDBFromTable) expression); - } else if (expression instanceof CnosDBSubquery) { - visit((CnosDBSubquery) expression); - } else if (expression instanceof CnosDBLikeOperation) { - visit((CnosDBLikeOperation) expression); - } else { - throw new AssertionError(expression); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAggregate.java b/src/sqlancer/cnosdb/ast/CnosDBAggregate.java deleted file mode 100644 index df30717b4..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBAggregate.java +++ /dev/null @@ -1,113 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBBugs; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; -import sqlancer.common.ast.FunctionNode; - -public class CnosDBAggregate extends FunctionNode - implements CnosDBExpression { - - public CnosDBAggregate(List args, CnosDBAggregateFunction func) { - super(func, args); - } - - public enum CnosDBAggregateFunction { - AVG(CnosDBDataType.DOUBLE), - MAX(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, - CnosDBDataType.UINT), - MIN(CnosDBDataType.DOUBLE, CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP, - CnosDBDataType.UINT), - COUNT(CnosDBDataType.INT) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.getRandomType() }; - } - }, - SUM(CnosDBDataType.INT, CnosDBDataType.DOUBLE, CnosDBDataType.UINT), APPROX_MEDIAN(CnosDBDataType.DOUBLE), - - VAR(CnosDBDataType.DOUBLE), VAR_SAMP(CnosDBDataType.DOUBLE), VAR_POP(CnosDBDataType.DOUBLE), - STDDEV(CnosDBDataType.DOUBLE), STDDEV_SAMP(CnosDBDataType.DOUBLE), STDDEV_POP(CnosDBDataType.DOUBLE), - COVAR(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - COVAR_SAMP(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.INT }; - } - }, - CORR(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - COVAR_POP(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - - APPROX_PERCENTILE_CONT(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - APPROX_PERCENTILE_CONT_WITH_WEIGHT(CnosDBDataType.DOUBLE) { - @Override - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE }; - } - }, - APPROX_DISTINCT(CnosDBDataType.UINT), GROUPING(CnosDBDataType.INT), ARRAY_AGG(CnosDBDataType.STRING); - - private final CnosDBDataType[] supportedReturnTypes; - - CnosDBAggregateFunction(CnosDBDataType... supportedReturnTypes) { - this.supportedReturnTypes = supportedReturnTypes.clone(); - } - - public static List getAggregates(CnosDBDataType type) { - List res = Stream.of(values()).filter(p -> p.supportsReturnType(type)) - .collect(Collectors.toList()); - if (CnosDBBugs.BUG786) { - res.removeAll(List.of(VAR, VAR_POP, VAR_SAMP, STDDEV, STDDEV_POP, STDDEV_SAMP, CORR, COVAR, COVAR_POP, - COVAR_SAMP, APPROX_PERCENTILE_CONT_WITH_WEIGHT, APPROX_DISTINCT, APPROX_PERCENTILE_CONT, - APPROX_PERCENTILE_CONT_WITH_WEIGHT, GROUPING, ARRAY_AGG)); - } - - return res; - } - - public CnosDBDataType[] getArgsTypes(CnosDBDataType returnType) { - return new CnosDBDataType[] { returnType }; - } - - public boolean supportsReturnType(CnosDBDataType returnType) { - return Arrays.stream(supportedReturnTypes).anyMatch(t -> t == returnType) - || supportedReturnTypes.length == 0; - } - - public CnosDBDataType getRandomReturnType() { - if (supportedReturnTypes.length == 0) { - return Randomly.fromOptions(CnosDBDataType.getRandomType()); - } else { - return Randomly.fromOptions(supportedReturnTypes); - } - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBAlias.java b/src/sqlancer/cnosdb/ast/CnosDBAlias.java deleted file mode 100644 index 86bba199f..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBAlias.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.common.visitor.UnaryOperation; - -public class CnosDBAlias implements UnaryOperation, CnosDBExpression { - - private final CnosDBExpression expr; - private final String alias; - - public CnosDBAlias(CnosDBExpression expr, String alias) { - this.expr = expr; - this.alias = alias; - } - - @Override - public CnosDBExpression getExpression() { - return expr; - } - - @Override - public String getOperatorRepresentation() { - return " as " + alias; - } - - @Override - public OperatorKind getOperatorKind() { - return OperatorKind.POSTFIX; - } - - @Override - public boolean omitBracketsWhenPrinting() { - return true; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java deleted file mode 100644 index d0addced1..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBetweenOperation.java +++ /dev/null @@ -1,34 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public final class CnosDBBetweenOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final CnosDBExpression left; - private final CnosDBExpression right; - - public CnosDBBetweenOperation(CnosDBExpression expr, CnosDBExpression left, CnosDBExpression right) { - this.expr = expr; - this.left = left; - this.right = right; - } - - public CnosDBExpression getExpr() { - return expr; - } - - public CnosDBExpression getLeft() { - return left; - } - - public CnosDBExpression getRight() { - return right; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java deleted file mode 100644 index acf3e93d5..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBinaryArithmeticOperation.java +++ /dev/null @@ -1,69 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; -import sqlancer.common.ast.BinaryOperatorNode; - -public class CnosDBBinaryArithmeticOperation extends BinaryOperatorNode - implements CnosDBExpression { - - public CnosDBBinaryArithmeticOperation(CnosDBExpression left, CnosDBExpression right, CnosDBBinaryOperator op) { - super(left, right, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.INT; - } - - public enum CnosDBBinaryOperator implements BinaryOperatorNode.Operator { - - ADDITION("+") { - }, - SUBTRACTION("-") { - }, - MULTIPLICATION("*") { - }, - DIVISION("/") { - - }, - MODULO("%") { - }, - EXPONENTIATION("^") { - }; - - private final String textRepresentation; - - CnosDBBinaryOperator(String textRepresentation) { - this.textRepresentation = textRepresentation; - } - - public static CnosDBBinaryOperator getRandom(CnosDBDataType dataType) { - List ops = new ArrayList<>(Arrays.asList(values())); - switch (dataType) { - case DOUBLE: - case UINT: - case STRING: - ops.remove(EXPONENTIATION); - ops.remove(MODULO); - break; - default: - break; - } - - return Randomly.fromList(ops); - } - - @Override - public String getTextRepresentation() { - return textRepresentation; - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java deleted file mode 100644 index af38849c9..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBinaryComparisonOperation.java +++ /dev/null @@ -1,57 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator; -import sqlancer.common.ast.BinaryOperatorNode; - -public class CnosDBBinaryComparisonOperation - extends BinaryOperatorNode implements CnosDBExpression { - - public CnosDBBinaryComparisonOperation(CnosDBExpression left, CnosDBExpression right, - CnosDBBinaryComparisonOperator op) { - super(left, right, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - public enum CnosDBBinaryComparisonOperator implements BinaryOperatorNode.Operator { - EQUALS("=") { - }, - IS_DISTINCT("IS DISTINCT FROM") { - }, - IS_NOT_DISTINCT("IS NOT DISTINCT FROM") { - }, - NOT_EQUALS("!=") { - }, - LESS("<") { - }, - LESS_EQUALS("<=") { - }, - GREATER(">") { - }, - GREATER_EQUALS(">=") { - - }; - - private final String textRepresentation; - - CnosDBBinaryComparisonOperator(String textRepresentation) { - this.textRepresentation = textRepresentation; - } - - public static CnosDBBinaryComparisonOperator getRandom() { - return Randomly.fromOptions(CnosDBBinaryComparisonOperator.values()); - } - - @Override - public String getTextRepresentation() { - return textRepresentation; - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java b/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java deleted file mode 100644 index bad8a3b75..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBBinaryLogicalOperation.java +++ /dev/null @@ -1,33 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; -import sqlancer.common.ast.BinaryOperatorNode; - -public class CnosDBBinaryLogicalOperation extends BinaryOperatorNode - implements CnosDBExpression { - - public CnosDBBinaryLogicalOperation(CnosDBExpression left, CnosDBExpression right, BinaryLogicalOperator op) { - super(left, right, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - public enum BinaryLogicalOperator implements BinaryOperatorNode.Operator { - AND, OR; - - public static BinaryLogicalOperator getRandom() { - return Randomly.fromOptions(values()); - } - - @Override - public String getTextRepresentation() { - return toString(); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java b/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java deleted file mode 100644 index 41db62d81..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBCastOperation.java +++ /dev/null @@ -1,60 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -import sqlancer.cnosdb.CnosDBCompoundDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBCastOperation implements CnosDBExpression { - - private final CnosDBExpression expression; - private final CnosDBCompoundDataType type; - - public CnosDBCastOperation(CnosDBExpression expression, CnosDBCompoundDataType type) { - if (expression == null) { - throw new AssertionError(); - } - this.expression = expression; - this.type = type; - } - - public static List canCastTo(CnosDBDataType dataType) { - List options = new ArrayList<>(Arrays.asList(CnosDBDataType.values())); - - switch (dataType) { - case UINT: - case BOOLEAN: - case DOUBLE: - options.remove(CnosDBDataType.TIMESTAMP); - break; - case TIMESTAMP: - options.remove(CnosDBDataType.BOOLEAN); - options.remove(CnosDBDataType.UINT); - options.remove(CnosDBDataType.DOUBLE); - break; - default: - break; - } - return options; - } - - @Override - public CnosDBDataType getExpressionType() { - return type.getDataType(); - } - - public CnosDBExpression getExpression() { - return expression; - } - - public CnosDBDataType getType() { - return type.getDataType(); - } - - public CnosDBCompoundDataType getCompoundType() { - return type; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java b/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java deleted file mode 100644 index f90b6120f..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBColumnValue.java +++ /dev/null @@ -1,27 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBColumnValue implements CnosDBExpression { - - private final CnosDBColumn c; - - public CnosDBColumnValue(CnosDBColumn c) { - this.c = c; - } - - public static CnosDBColumnValue create(CnosDBColumn c) { - return new CnosDBColumnValue(c); - } - - @Override - public CnosDBDataType getExpressionType() { - return c.getType(); - } - - public CnosDBColumn getColumn() { - return c; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java b/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java deleted file mode 100644 index 6821f83b8..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBConcatOperation.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryNode; - -public class CnosDBConcatOperation extends BinaryNode implements CnosDBExpression { - - public CnosDBConcatOperation(CnosDBExpression left, CnosDBExpression right) { - super(left, right); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.STRING; - } - - @Override - public String getOperatorRepresentation() { - return "||"; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBConstant.java b/src/sqlancer/cnosdb/ast/CnosDBConstant.java deleted file mode 100644 index 42ecd3908..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBConstant.java +++ /dev/null @@ -1,520 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.math.BigDecimal; -import java.text.SimpleDateFormat; -import java.util.Date; - -import sqlancer.IgnoreMeException; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public abstract class CnosDBConstant implements CnosDBExpression { - - public static CnosDBConstant createNullConstant() { - return new CnosDBNullConstant(); - } - - public static CnosDBConstant createIntConstant(long val) { - return new IntConstant(val, false); - } - - public static CnosDBConstant createBooleanConstant(boolean val) { - return new BooleanConstant(val); - } - - public static CnosDBConstant createFalse() { - return createBooleanConstant(false); - } - - public static CnosDBConstant createTrue() { - return createBooleanConstant(true); - } - - public static CnosDBConstant createStringConstant(String string) { - return new StringConstant(string); - } - - public static CnosDBConstant createDoubleConstant(double val) { - return new DoubleConstant(val); - } - - public static CnosDBConstant createUintConstant(long val) { - return new IntConstant(val, true); - } - - public static CnosDBConstant createTimeStampConstant(long val) { - return new TimeStampConstant(val); - } - - public abstract String getTextRepresentation(); - - public String asString() { - throw new UnsupportedOperationException(this.toString()); - } - - public boolean isString() { - return false; - } - - public boolean isNull() { - return false; - } - - public boolean asBoolean() { - throw new UnsupportedOperationException(this.toString()); - } - - public long asInt() { - throw new UnsupportedOperationException(this.toString()); - } - - public double asDouble() { - throw new UnsupportedOperationException(this.toString()); - } - - public boolean isBoolean() { - return false; - } - - public abstract CnosDBConstant isEquals(CnosDBConstant rightVal); - - public boolean isInt() { - return false; - } - - protected abstract CnosDBConstant isLessThan(CnosDBConstant rightVal); - - @Override - public String toString() { - return getTextRepresentation(); - } - - public abstract CnosDBConstant cast(CnosDBDataType type); - - public static class BooleanConstant extends CnosDBConstant { - - private final boolean value; - - public BooleanConstant(boolean value) { - this.value = value; - } - - @Override - public String getTextRepresentation() { - return value ? "TRUE" : "FALSE"; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - @Override - public boolean asBoolean() { - return value; - } - - @Override - public boolean isBoolean() { - return true; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return CnosDBConstant.createBooleanConstant(value == rightVal.asBoolean()); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(value == rightVal.cast(CnosDBDataType.BOOLEAN).asBoolean()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isString()) { - return isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); - } else { - assert rightVal.isBoolean(); - return CnosDBConstant.createBooleanConstant((value ? 1 : 0) < (rightVal.asBoolean() ? 1 : 0)); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - switch (type) { - case BOOLEAN: - return this; - case INT: - return CnosDBConstant.createIntConstant(value ? 1 : 0); - case UINT: - return CnosDBConstant.createUintConstant(value ? 1 : 0); - case STRING: - return CnosDBConstant.createStringConstant(value ? "true" : "false"); - default: - return null; - } - } - - } - - public static class CnosDBNullConstant extends CnosDBConstant { - - @Override - public String getTextRepresentation() { - return "NULL"; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - - @Override - public boolean isNull() { - return true; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - return CnosDBConstant.createNullConstant(); - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - return CnosDBConstant.createNullConstant(); - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - return CnosDBConstant.createNullConstant(); - } - } - - public static class StringConstant extends CnosDBConstant { - - private final String value; - - public StringConstant(String value) { - this.value = value; - } - - @Override - public String getTextRepresentation() { - return String.format("'%s'", value.replace("'", "''")); - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isInt()) { - return cast(CnosDBDataType.INT).isEquals(rightVal.cast(CnosDBDataType.INT)); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal.cast(CnosDBDataType.BOOLEAN)); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(value.contentEquals(rightVal.asString())); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isInt()) { - return cast(CnosDBDataType.INT).isLessThan(rightVal.cast(CnosDBDataType.INT)); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal.cast(CnosDBDataType.BOOLEAN)); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(value.compareTo(rightVal.asString()) < 0); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - if (type == CnosDBDataType.STRING) { - return this; - } - String s = value.trim(); - switch (type) { - case BOOLEAN: - try { - return CnosDBConstant.createBooleanConstant(Long.parseLong(s) != 0); - } catch (NumberFormatException ignored) { - } - switch (s.toUpperCase()) { - case "T": - case "TR": - case "TRU": - case "TRUE": - case "1": - case "YES": - case "YE": - case "Y": - case "ON": - return CnosDBConstant.createTrue(); - case "F": - case "FA": - case "FAL": - case "FALS": - case "FALSE": - case "N": - case "NO": - case "OF": - case "OFF": - default: - return CnosDBConstant.createFalse(); - } - case INT: - try { - return CnosDBConstant.createIntConstant(Long.parseLong(s)); - } catch (NumberFormatException e) { - return CnosDBConstant.createIntConstant(-1); - } - case UINT: - try { - return CnosDBConstant.createUintConstant(Long.parseUnsignedLong(s)); - } catch (NumberFormatException e) { - return CnosDBConstant.createUintConstant(0); - } - case DOUBLE: - try { - return CnosDBConstant.createDoubleConstant(Double.parseDouble(s)); - } catch (NumberFormatException e) { - return CnosDBConstant.createDoubleConstant(0.0); - } - - default: - return null; - } - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.STRING; - } - - @Override - public boolean isString() { - return true; - } - - @Override - public String asString() { - return value; - } - - } - - public static class IntConstant extends CnosDBConstant { - - private final long val; - private final boolean unsigned; - - public IntConstant(long val, boolean unsigned) { - this.val = val; - this.unsigned = unsigned; - } - - @Override - public String getTextRepresentation() { - if (unsigned) { - return Long.toUnsignedString(val); - } else { - return String.valueOf(val); - } - } - - @Override - public CnosDBDataType getExpressionType() { - if (unsigned) { - return CnosDBDataType.UINT; - } - return CnosDBDataType.INT; - } - - @Override - public long asInt() { - return val; - } - - @Override - public double asDouble() { - return val; - } - - @Override - public boolean isInt() { - return true; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); - } else if (rightVal.isInt()) { - return CnosDBConstant.createBooleanConstant(val == rightVal.asInt()); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.INT).asInt()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isInt()) { - return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); - } else if (rightVal.isBoolean()) { - throw new AssertionError(rightVal); - } else if (rightVal.getExpressionType() == CnosDBDataType.UINT) { - return CnosDBConstant.createBooleanConstant(Long.compareUnsigned(val, rightVal.asInt()) < 0); - } else if (rightVal.isString()) { - return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.INT).asInt()); - } else { - throw new IgnoreMeException(); - } - - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - switch (type) { - case BOOLEAN: - return CnosDBConstant.createBooleanConstant(val != 0); - case INT: - return CnosDBConstant.createIntConstant(val); - case STRING: - return CnosDBConstant.createStringConstant(String.valueOf(val)); - case UINT: - return CnosDBConstant.createUintConstant(val); - case DOUBLE: - return CnosDBConstant.createDoubleConstant(val); - default: - return null; - } - } - } - - public static class TimeStampConstant extends CnosDBConstant { - final long val; - - TimeStampConstant(long time) { - val = time; - } - - @Override - public String getTextRepresentation() { - return "CAST (" + val + " AS TIMESTAMP)"; - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return createNullConstant(); - } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { - return createBooleanConstant(val == rightVal.asInt()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.getExpressionType() == CnosDBDataType.TIMESTAMP) { - return CnosDBConstant.createBooleanConstant(val < rightVal.asInt()); - } else { - throw new AssertionError(rightVal); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - switch (type) { - case INT: - return createIntConstant(val); - case STRING: - final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss"); - return CnosDBConstant.createStringConstant(dateFormat.format(new Date(val))); - default: - return null; - } - } - - @Override - public long asInt() { - return val; - } - - } - - public static class DoubleConstant extends CnosDBConstant { - - private final double val; - - public DoubleConstant(double val) { - this.val = val; - } - - @Override - public String getTextRepresentation() { - if (Double.isFinite(val)) { - BigDecimal bigDecimal = new BigDecimal(val); - return bigDecimal.toPlainString(); - } else { - return String.valueOf(0.0); - } - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.DOUBLE; - } - - @Override - public boolean isNull() { - return false; - } - - @Override - protected CnosDBConstant isLessThan(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isLessThan(rightVal); - } else { - return CnosDBConstant.createBooleanConstant(val < rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); - } - } - - @Override - public CnosDBConstant isEquals(CnosDBConstant rightVal) { - if (rightVal.isNull()) { - return CnosDBConstant.createNullConstant(); - } else if (rightVal.isBoolean()) { - return cast(CnosDBDataType.BOOLEAN).isEquals(rightVal); - } else { - return CnosDBConstant.createBooleanConstant(val == rightVal.cast(CnosDBDataType.DOUBLE).asDouble()); - } - } - - @Override - public CnosDBConstant cast(CnosDBDataType type) { - return null; - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBExpression.java b/src/sqlancer/cnosdb/ast/CnosDBExpression.java deleted file mode 100644 index 63997a0f5..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBExpression.java +++ /dev/null @@ -1,14 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public interface CnosDBExpression { - - default CnosDBDataType getExpressionType() { - return null; - } - - default CnosDBConstant getExpectedValue() { - throw new AssertionError("Not impl"); - } -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunction.java b/src/sqlancer/cnosdb/ast/CnosDBFunction.java deleted file mode 100644 index 7a35d703e..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBFunction.java +++ /dev/null @@ -1,30 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBFunction implements CnosDBExpression { - - private final String func; - private final CnosDBExpression[] args; - private final CnosDBDataType returnType; - - public CnosDBFunction(CnosDBFunctionWithUnknownResult f, CnosDBDataType returnType, CnosDBExpression... args) { - this.func = f.getName(); - this.returnType = returnType; - this.args = args.clone(); - } - - public String getFunctionName() { - return func; - } - - public CnosDBExpression[] getArguments() { - return args.clone(); - } - - @Override - public CnosDBDataType getExpressionType() { - return returnType; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java b/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java deleted file mode 100644 index 485f2309d..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBFunctionWithUnknownResult.java +++ /dev/null @@ -1,104 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import sqlancer.cnosdb.CnosDBBugs; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; - -public enum CnosDBFunctionWithUnknownResult { - - // String functions - ASCII("ascii", CnosDBDataType.INT, CnosDBDataType.STRING), - BTRIM("btrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - CHAR_LENGTH("char_length", CnosDBDataType.INT, CnosDBDataType.STRING), - CHARACTER_LENGTH("character_length", CnosDBDataType.INT, CnosDBDataType.STRING), - CONCAT("concat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - CONCAT_WS("concat_ws", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - CHR("chr", CnosDBDataType.STRING, CnosDBDataType.INT), - BIT_LENGTH("bit_length", CnosDBDataType.INT, CnosDBDataType.STRING), - INITCAP("initcap", CnosDBDataType.STRING, CnosDBDataType.STRING), - - LEFT("left", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - LENGTH("length", CnosDBDataType.UINT, CnosDBDataType.STRING), - LOWER("lower", CnosDBDataType.STRING, CnosDBDataType.STRING), - UPPER("upper", CnosDBDataType.STRING, CnosDBDataType.STRING), - LPAD3("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), - LPAD2("lpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - RPAD3("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.STRING), - RPAD2("rpad", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - LTRIM("ltrim", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - OCTET_LENGTH("octet_length", CnosDBDataType.INT, CnosDBDataType.STRING), - // REPEAT("repeat", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - REPLACE("replace", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - REVERSE("reverse", CnosDBDataType.STRING, CnosDBDataType.STRING), - RIGHT("right", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - RTRIM("rtrim", CnosDBDataType.STRING, CnosDBDataType.STRING), - SPLIT_PART("split_part", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT), - STARTS_WITH("starts_with", CnosDBDataType.BOOLEAN, CnosDBDataType.STRING, CnosDBDataType.STRING), - STRPOS("strpos", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.STRING), - SUBSTR("substr", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.INT, CnosDBDataType.INT), - TRANSLATE("translate", CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING, CnosDBDataType.STRING), - MD5("md5", CnosDBDataType.STRING, CnosDBDataType.STRING), - // mathematical functions - ABS("abs", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - CEIL("ceil", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - EXP("exp", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), LN("ln", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - LOG2("log2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - LOG10("log10", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - POWER("power", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ROUND("round", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - TRUNC("trunc", CnosDBDataType.DOUBLE, CnosDBDataType.INT), - FLOOR("floor", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - SIGNUM("signum", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ACOS("acos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ASIN("asin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - ATAN2("atan2", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - COS("cos", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), SIN("sin", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - SQRT("sqrt", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - TAN("tan", CnosDBDataType.DOUBLE, CnosDBDataType.DOUBLE), - DATE_PART("date_part", CnosDBDataType.INT, CnosDBDataType.STRING, CnosDBDataType.TIMESTAMP), - TO_TIMESTAMP("to_timestamp", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), - TO_TIMESTAMP_MILLIS("to_timestamp_millis", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), - TO_TIMESTAMP_MICROS("to_timestamp_micros", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT), - TO_TIMESTAMP_SECONDS("to_timestamp_seconds", CnosDBDataType.TIMESTAMP, CnosDBDataType.INT); - - private final String functionName; - private final CnosDBDataType returnType; - private final CnosDBDataType[] argTypes; - - CnosDBFunctionWithUnknownResult(String functionName, CnosDBDataType returnType, CnosDBDataType... indexType) { - this.functionName = functionName; - this.returnType = returnType; - this.argTypes = indexType.clone(); - - } - - public static List getSupportedFunctions(CnosDBDataType type) { - List res = Stream.of(values()) - .filter(function -> function.isCompatibleWithReturnType(type)).collect(Collectors.toList()); - if (CnosDBBugs.BUG3547) { - res.removeAll(List.of(TO_TIMESTAMP, TO_TIMESTAMP_MICROS, TO_TIMESTAMP_MILLIS, TO_TIMESTAMP_SECONDS)); - } - return res; - } - - public boolean isCompatibleWithReturnType(CnosDBDataType t) { - return t == returnType; - } - - public CnosDBExpression[] getArguments(CnosDBDataType ignore, CnosDBExpressionGenerator gen, int depth) { - CnosDBExpression[] args = new CnosDBExpression[argTypes.length]; - for (int i = 0; i < args.length; i++) { - args[i] = gen.generateExpression(depth, argTypes[i]); - } - return args; - } - - public String getName() { - return functionName; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBInOperation.java b/src/sqlancer/cnosdb/ast/CnosDBInOperation.java deleted file mode 100644 index c0ffd34ed..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBInOperation.java +++ /dev/null @@ -1,35 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.List; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBInOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final List listElements; - private final boolean isTrue; - - public CnosDBInOperation(CnosDBExpression expr, List listElements, boolean isTrue) { - this.expr = expr; - this.listElements = listElements; - this.isTrue = isTrue; - } - - public CnosDBExpression getExpr() { - return expr; - } - - public List getListElements() { - return listElements; - } - - public boolean isTrue() { - return isTrue; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBJoin.java b/src/sqlancer/cnosdb/ast/CnosDBJoin.java deleted file mode 100644 index eea88466f..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBJoin.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBJoin implements CnosDBExpression { - - private final CnosDBExpression tableReference; - private final CnosDBExpression onClause; - private final CnosDBJoinType type; - - public CnosDBJoin(CnosDBExpression tableReference, CnosDBExpression onClause, CnosDBJoinType type) { - this.tableReference = tableReference; - this.onClause = onClause; - this.type = type; - } - - public CnosDBExpression getTableReference() { - return tableReference; - } - - public CnosDBExpression getOnClause() { - return onClause; - } - - public CnosDBJoinType getType() { - return type; - } - - @Override - public CnosDBDataType getExpressionType() { - throw new AssertionError(); - } - - public enum CnosDBJoinType { - INNER, LEFT, RIGHT, FULL; - // now not support - // CROSS; - - public static CnosDBJoinType getRandom() { - return Randomly.fromOptions(values()); - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java b/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java deleted file mode 100644 index 616cd39ee..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBLikeOperation.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryNode; - -public class CnosDBLikeOperation extends BinaryNode implements CnosDBExpression { - - public CnosDBLikeOperation(CnosDBExpression left, CnosDBExpression right) { - super(left, right); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - @Override - public String getOperatorRepresentation() { - return "LIKE"; - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java b/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java deleted file mode 100644 index de5812d76..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBOrderByTerm.java +++ /dev/null @@ -1,37 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBOrderByTerm implements CnosDBExpression { - - private final CnosDBOrder order; - private final CnosDBExpression expr; - - public CnosDBOrderByTerm(CnosDBExpression expr, CnosDBOrder order) { - this.expr = expr; - this.order = order; - } - - public CnosDBOrder getOrder() { - return order; - } - - public CnosDBExpression getExpr() { - return expr; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - - public enum CnosDBOrder { - ASC, DESC; - - public static CnosDBOrder getRandomOrder() { - return Randomly.fromOptions(CnosDBOrder.values()); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java deleted file mode 100644 index f37621f44..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBPostfixOperation.java +++ /dev/null @@ -1,97 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryOperatorNode.Operator; - -public class CnosDBPostfixOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final String operatorTextRepresentation; - - public CnosDBPostfixOperation(CnosDBExpression expr, PostfixOperator op) { - this.expr = expr; - this.operatorTextRepresentation = Randomly.fromOptions(op.textRepresentations); - } - - public static CnosDBExpression create(CnosDBExpression expr, PostfixOperator op) { - return new CnosDBPostfixOperation(expr, op); - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - public String getOperatorTextRepresentation() { - return operatorTextRepresentation; - } - - public CnosDBExpression getExpression() { - return expr; - } - - public enum PostfixOperator implements Operator { - IS_NULL("IS NULL"/* , "ISNULL" */) { - @Override - public CnosDBDataType[] getInputDataTypes() { - return CnosDBDataType.values(); - } - - }, - IS_UNKNOWN("IS UNKNOWN") { - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - }, - - IS_NOT_NULL("IS NOT NULL"/* "NOTNULL" */) { - - @Override - public CnosDBDataType[] getInputDataTypes() { - return CnosDBDataType.values(); - } - - }, - IS_NOT_UNKNOWN("IS NOT UNKNOWN") { - - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - }, - IS_TRUE("IS TRUE") { - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - - }, - IS_FALSE("IS FALSE") { - @Override - public CnosDBDataType[] getInputDataTypes() { - return new CnosDBDataType[] { CnosDBDataType.BOOLEAN }; - } - - }; - - private final String[] textRepresentations; - - PostfixOperator(String... textRepresentations) { - this.textRepresentations = textRepresentations.clone(); - } - - public static PostfixOperator getRandom() { - return Randomly.fromOptions(values()); - } - - public abstract CnosDBDataType[] getInputDataTypes(); - - @Override - public String getTextRepresentation() { - return toString(); - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java b/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java deleted file mode 100644 index 241fab89a..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBPostfixText.java +++ /dev/null @@ -1,29 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBPostfixText implements CnosDBExpression { - - private final CnosDBExpression expr; - private final String text; - private final CnosDBDataType type; - - public CnosDBPostfixText(CnosDBExpression expr, String text, CnosDBDataType type) { - this.expr = expr; - this.text = text; - this.type = type; - } - - public CnosDBExpression getExpr() { - return expr; - } - - public String getText() { - return text; - } - - @Override - public CnosDBDataType getExpressionType() { - return type; - } -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java b/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java deleted file mode 100644 index db37f0089..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBPrefixOperation.java +++ /dev/null @@ -1,73 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.common.ast.BinaryOperatorNode.Operator; - -public class CnosDBPrefixOperation implements CnosDBExpression { - - private final CnosDBExpression expr; - private final PrefixOperator op; - - public CnosDBPrefixOperation(CnosDBExpression expr, PrefixOperator op) { - this.expr = expr; - this.op = op; - } - - @Override - public CnosDBDataType getExpressionType() { - return op.getExpressionType(); - } - - public CnosDBDataType[] getInputDataTypes() { - return op.dataTypes; - } - - public String getTextRepresentation() { - return op.textRepresentation; - } - - public CnosDBExpression getExpression() { - return expr; - } - - public enum PrefixOperator implements Operator { - NOT("NOT", CnosDBDataType.BOOLEAN) { - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - - }, - UNARY_PLUS("+", CnosDBDataType.INT) { - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.INT; - } - - }, - UNARY_MINUS("-", CnosDBDataType.INT) { - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.INT; - } - - }; - - private final String textRepresentation; - private final CnosDBDataType[] dataTypes; - - PrefixOperator(String textRepresentation, CnosDBDataType... dataTypes) { - this.textRepresentation = textRepresentation; - this.dataTypes = dataTypes.clone(); - } - - public abstract CnosDBDataType getExpressionType(); - - @Override - public String getTextRepresentation() { - return toString(); - } - - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSelect.java b/src/sqlancer/cnosdb/ast/CnosDBSelect.java deleted file mode 100644 index 0db657f19..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBSelect.java +++ /dev/null @@ -1,102 +0,0 @@ -package sqlancer.cnosdb.ast; - -import java.util.Collections; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.common.ast.SelectBase; - -public class CnosDBSelect extends SelectBase implements CnosDBExpression { - - private SelectType selectOption = SelectType.ALL; - private List joinClauses = Collections.emptyList(); - private CnosDBExpression distinctOnClause; - - public void setSelectType(SelectType fromOptions) { - this.setSelectOption(fromOptions); - } - - public SelectType getSelectOption() { - return selectOption; - } - - public void setSelectOption(SelectType fromOptions) { - this.selectOption = fromOptions; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - - public List getJoinClauses() { - return joinClauses; - } - - public void setJoinClauses(List joinStatements) { - this.joinClauses = joinStatements; - - } - - public CnosDBExpression getDistinctOnClause() { - return distinctOnClause; - } - - public void setDistinctOnClause(CnosDBExpression distinctOnClause) { - if (selectOption != SelectType.DISTINCT) { - throw new IllegalArgumentException(); - } - this.distinctOnClause = distinctOnClause; - } - - public enum SelectType { - DISTINCT, ALL; - - public static SelectType getRandom() { - return Randomly.fromOptions(values()); - } - } - - public static class CnosDBFromTable implements CnosDBExpression { - private final CnosDBTable t; - - public CnosDBFromTable(CnosDBTable t) { - this.t = t; - } - - public CnosDBTable getTable() { - return t; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - } - - public static class CnosDBSubquery implements CnosDBExpression { - private final CnosDBSelect s; - private final String name; - - public CnosDBSubquery(CnosDBSelect s, String name) { - this.s = s; - this.name = name; - } - - public CnosDBSelect getSelect() { - return s; - } - - public String getName() { - return name; - } - - @Override - public CnosDBDataType getExpressionType() { - return null; - } - } - -} diff --git a/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java b/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java deleted file mode 100644 index 9e3467ada..000000000 --- a/src/sqlancer/cnosdb/ast/CnosDBSimilarTo.java +++ /dev/null @@ -1,28 +0,0 @@ -package sqlancer.cnosdb.ast; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public class CnosDBSimilarTo implements CnosDBExpression { - - private final CnosDBExpression string; - private final CnosDBExpression similarTo; - - public CnosDBSimilarTo(CnosDBExpression string, CnosDBExpression similarTo) { - this.string = string; - this.similarTo = similarTo; - } - - public CnosDBExpression getString() { - return string; - } - - public CnosDBExpression getSimilarTo() { - return similarTo; - } - - @Override - public CnosDBDataType getExpressionType() { - return CnosDBDataType.BOOLEAN; - } - -} diff --git a/src/sqlancer/cnosdb/client/CnosDBClient.java b/src/sqlancer/cnosdb/client/CnosDBClient.java deleted file mode 100644 index ccc9dcc16..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBClient.java +++ /dev/null @@ -1,110 +0,0 @@ -package sqlancer.cnosdb.client; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; -import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; - -import org.apache.commons.codec.binary.Base64; -import org.apache.http.HttpHeaders; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; - -import com.arangodb.internal.util.IOUtils; - -public class CnosDBClient { - private final String userName; - private final String password; - private final String host; - private final int port; - - private final String database; - private final CloseableHttpClient client; - - public CnosDBClient(String host, int port, String userName, String password, String database) { - this.host = host; - this.port = port; - this.userName = userName; - this.password = password; - this.database = database; - this.client = HttpClientBuilder.create().build(); - } - - private String url() { - return "http://" + host + ":" + port + "/api/v1/"; - } - - public String ping() throws Exception { - HttpGet httpGet = new HttpGet(this.url() + "ping"); - httpGet.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); - CloseableHttpResponse resp = client.execute(httpGet); - - String content = IOUtils.toString(resp.getEntity().getContent()); - resp.close(); - return content; - } - - public CnosDBResultSet executeQuery(String query) throws Exception { - HttpUriRequest request = createRequest(query); - CloseableHttpResponse resp = client.execute(request); - String text = IOUtils.toString(resp.getEntity().getContent()); - if (resp.getStatusLine().getStatusCode() != 200) { - resp.close(); - throw new CnosDBException(database + ":" + query + ";\n" + text); - } - resp.close(); - InputStream stream = new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8)); - - return new CnosDBResultSet(new InputStreamReader(stream)); - } - - public boolean execute(String query) throws Exception { - HttpUriRequest request = createRequest(query); - CloseableHttpResponse resp = client.execute(request); - if (resp.getStatusLine().getStatusCode() != 200) { - String res = IOUtils.toString(resp.getEntity().getContent()); - resp.close(); - throw new CnosDBException(query + res); - } - resp.close(); - return true; - } - - public void close() throws IOException { - client.close(); - } - - public String getDatabase() { - return this.database; - } - - private String getAuth() { - String auth = userName + ":" + password; - byte[] encodedAuth = Base64.encodeBase64(auth.getBytes(StandardCharsets.ISO_8859_1)); - return "Basic " + new String(encodedAuth); - - } - - private HttpUriRequest createRequest(String query) throws URISyntaxException, UnsupportedEncodingException { - - URIBuilder builder = new URIBuilder(this.url() + "sql"); - builder.setParameter("db", database); - builder.setParameter("pretty", "true"); - HttpPost httpPost = new HttpPost(builder.build()); - - httpPost.setHeader(HttpHeaders.AUTHORIZATION, getAuth()); - StringEntity stringEntity = new StringEntity(query); - httpPost.setEntity(stringEntity); - return httpPost; - } - -} diff --git a/src/sqlancer/cnosdb/client/CnosDBConnection.java b/src/sqlancer/cnosdb/client/CnosDBConnection.java deleted file mode 100644 index 9277f203b..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBConnection.java +++ /dev/null @@ -1,27 +0,0 @@ -package sqlancer.cnosdb.client; - -import java.io.IOException; - -import sqlancer.SQLancerDBConnection; - -public class CnosDBConnection implements SQLancerDBConnection { - private final CnosDBClient client; - - public CnosDBConnection(CnosDBClient client) { - this.client = client; - } - - @Override - public String getDatabaseVersion() throws Exception { - return client.ping(); - } - - public CnosDBClient getClient() { - return client; - } - - @Override - public void close() throws IOException { - client.close(); - } -} diff --git a/src/sqlancer/cnosdb/client/CnosDBException.java b/src/sqlancer/cnosdb/client/CnosDBException.java deleted file mode 100644 index a1055e90b..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBException.java +++ /dev/null @@ -1,9 +0,0 @@ -package sqlancer.cnosdb.client; - -public class CnosDBException extends RuntimeException { - private static final long serialVersionUID = 1L; - - CnosDBException(String message) { - super(message); - } -} diff --git a/src/sqlancer/cnosdb/client/CnosDBResultSet.java b/src/sqlancer/cnosdb/client/CnosDBResultSet.java deleted file mode 100644 index 877b6ba5d..000000000 --- a/src/sqlancer/cnosdb/client/CnosDBResultSet.java +++ /dev/null @@ -1,52 +0,0 @@ -package sqlancer.cnosdb.client; - -import java.io.Reader; -import java.sql.SQLException; -import java.util.Iterator; - -import org.apache.commons.csv.CSVFormat; -import org.apache.commons.csv.CSVRecord; - -import sqlancer.IgnoreMeException; - -public class CnosDBResultSet { - private final Iterator records; - private CSVRecord next; - - public CnosDBResultSet(Reader in) throws Exception { - Iterable records = CSVFormat.DEFAULT.builder().setHeader().setSkipHeaderRecord(true).build() - .parse(in); - this.records = records.iterator(); - } - - public void close() { - } - - public boolean next() throws SQLException { - if (records.hasNext()) { - next = records.next(); - return true; - } - return false; - } - - public int getInt(int i) throws SQLException { - return Integer.parseInt(next.get(i - 1)); - } - - public String getString(int i) throws SQLException { - return next.get(i - 1); - } - - public long getLong(int i) throws SQLException { - if (next.get(i - 1).isEmpty()) { - throw new IgnoreMeException(); - } - return Long.parseLong(next.get(i - 1)); - } - - // public boolean getBool(int i) throws Exception { - // return Boolean.parseBoolean(getString(i)); - // } - -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBCommon.java b/src/sqlancer/cnosdb/gen/CnosDBCommon.java deleted file mode 100644 index 6c7b0bba7..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBCommon.java +++ /dev/null @@ -1,31 +0,0 @@ -package sqlancer.cnosdb.gen; - -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; - -public final class CnosDBCommon { - - private CnosDBCommon() { - } - - public static void appendDataType(CnosDBDataType type, StringBuilder sb) throws AssertionError { - switch (type) { - case BOOLEAN: - sb.append("BOOLEAN"); - break; - case INT: - sb.append("BIGINT"); - break; - case STRING: - sb.append("STRING"); - break; - case DOUBLE: - sb.append("DOUBLE"); - break; - case UINT: - sb.append("BIGINT UNSIGNED"); - break; - default: - throw new AssertionError(type); - } - } -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java deleted file mode 100644 index 121f78254..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBExpressionGenerator.java +++ /dev/null @@ -1,461 +0,0 @@ -package sqlancer.cnosdb.gen; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBCompoundDataType; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; -import sqlancer.cnosdb.ast.CnosDBBetweenOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryArithmeticOperation.CnosDBBinaryOperator; -import sqlancer.cnosdb.ast.CnosDBBinaryComparisonOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation; -import sqlancer.cnosdb.ast.CnosDBBinaryLogicalOperation.BinaryLogicalOperator; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConcatOperation; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBFunction; -import sqlancer.cnosdb.ast.CnosDBFunctionWithUnknownResult; -import sqlancer.cnosdb.ast.CnosDBInOperation; -import sqlancer.cnosdb.ast.CnosDBLikeOperation; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm; -import sqlancer.cnosdb.ast.CnosDBOrderByTerm.CnosDBOrder; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; -import sqlancer.cnosdb.ast.CnosDBSimilarTo; -import sqlancer.common.gen.ExpressionGenerator; - -public class CnosDBExpressionGenerator implements ExpressionGenerator { - - private final int maxDepth; - - private final Randomly r; - - private List columns; - - private boolean allowAggregateFunctions; - - public CnosDBExpressionGenerator(CnosDBGlobalState globalState) { - this.r = globalState.getRandomly(); - this.maxDepth = globalState.getOptions().getMaxExpressionDepth(); - } - - public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, CnosDBDataType type) { - return new CnosDBExpressionGenerator(globalState).generateExpression(0, type); - } - - private static CnosDBCompoundDataType getCompoundDataType(CnosDBDataType type) { - return CnosDBCompoundDataType.create(type); - } - - public static CnosDBExpression generateConstant(Randomly r, CnosDBDataType type) { - if (Randomly.getBooleanWithSmallProbability()) { - return CnosDBConstant.createNullConstant(); - } - switch (type) { - case INT: - return CnosDBConstant.createIntConstant(r.getInteger()); - case UINT: - return CnosDBConstant.createUintConstant(r.getPositiveInteger()); - case TIMESTAMP: - return CnosDBConstant.createTimeStampConstant(r.getPositiveIntegerNotNull()); - case BOOLEAN: - return CnosDBConstant.createBooleanConstant(Randomly.getBoolean()); - case STRING: - return CnosDBConstant.createStringConstant(r.getString()); - case DOUBLE: - return CnosDBConstant.createDoubleConstant(r.getDouble()); - default: - throw new AssertionError(type); - } - } - - public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns, - CnosDBDataType type) { - return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0, type); - } - - public static CnosDBExpression generateExpression(CnosDBGlobalState globalState, List columns) { - return new CnosDBExpressionGenerator(globalState).setColumns(columns).generateExpression(0); - } - - public CnosDBExpressionGenerator setColumns(List columns) { - this.columns = columns; - return this; - } - - public CnosDBExpression generateExpression(int depth) { - return generateExpression(depth, CnosDBDataType.getRandomType()); - } - - public List generateOrderBy() { - List orderBys = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber(); i++) { - orderBys.add(new CnosDBOrderByTerm(CnosDBColumnValue.create(Randomly.fromList(columns)), - CnosDBOrder.getRandomOrder())); - } - return orderBys; - } - - private CnosDBExpression generateFunctionWithUnknownResult(int depth, CnosDBDataType type) { - List supportedFunctions = CnosDBFunctionWithUnknownResult - .getSupportedFunctions(type); - if (supportedFunctions.isEmpty()) { - throw new IgnoreMeException(); - } - CnosDBFunctionWithUnknownResult randomFunction = Randomly.fromList(supportedFunctions); - return new CnosDBFunction(randomFunction, type, randomFunction.getArguments(type, this, depth + 1)); - } - - private CnosDBExpression generateBooleanExpression(int depth) { - List validOptions = new ArrayList<>(Arrays.asList(BooleanExpression.values())); - BooleanExpression option = Randomly.fromList(validOptions); - switch (option) { - case POSTFIX_OPERATOR: - PostfixOperator random = PostfixOperator.getRandom(); - return CnosDBPostfixOperation - .create(generateExpression(depth + 1, Randomly.fromOptions(random.getInputDataTypes())), random); - case IN_OPERATION: - return inOperation(depth + 1); - case NOT: - return new CnosDBPrefixOperation(generateExpression(depth + 1, CnosDBDataType.BOOLEAN), PrefixOperator.NOT); - case BINARY_LOGICAL_OPERATOR: - CnosDBExpression first = generateExpression(depth + 1, CnosDBDataType.BOOLEAN); - int nr = Randomly.smallNumber() + 1; - for (int i = 0; i < nr; i++) { - first = new CnosDBBinaryLogicalOperation(first, generateExpression(depth + 1, CnosDBDataType.BOOLEAN), - BinaryLogicalOperator.getRandom()); - } - return first; - case BINARY_COMPARISON: - CnosDBDataType dataType = getMeaningfulType(); - return generateComparison(depth, dataType); - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.BOOLEAN); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.BOOLEAN); - case LIKE: - return new CnosDBLikeOperation(generateExpression(depth + 1, CnosDBDataType.STRING), - generateExpression(depth + 1, CnosDBDataType.STRING)); - case BETWEEN: - CnosDBDataType type = getMeaningfulType(); - return new CnosDBBetweenOperation(generateExpression(depth + 1, type), generateExpression(depth + 1, type), - generateExpression(depth + 1, type)); - case SIMILAR_TO: - return new CnosDBSimilarTo(generateExpression(depth + 1, CnosDBDataType.STRING), - generateExpression(depth + 1, CnosDBDataType.STRING)); - default: - throw new AssertionError(); - } - } - - private CnosDBDataType getMeaningfulType() { - // make it more likely that the expression does not only consist of constant - // expressions - if (Randomly.getBooleanWithSmallProbability() || columns == null || columns.isEmpty()) { - return CnosDBDataType.getRandomType(); - } else { - return Randomly.fromList(columns).getType(); - } - } - - private CnosDBExpression generateFunction(int depth, CnosDBDataType type) { - return generateFunctionWithUnknownResult(depth, type); - } - - private CnosDBExpression generateComparison(int depth, CnosDBDataType dataType) { - CnosDBExpression leftExpr = generateExpression(depth + 1, dataType); - CnosDBExpression rightExpr = generateExpression(depth + 1, dataType); - return getComparison(leftExpr, rightExpr); - } - - private CnosDBExpression getComparison(CnosDBExpression leftExpr, CnosDBExpression rightExpr) { - return new CnosDBBinaryComparisonOperation(leftExpr, rightExpr, - CnosDBBinaryComparisonOperation.CnosDBBinaryComparisonOperator.getRandom()); - } - - private CnosDBExpression inOperation(int depth) { - CnosDBDataType type = CnosDBDataType.getRandomType(); - CnosDBExpression leftExpr = generateExpression(depth + 1, type); - List rightExpr = new ArrayList<>(); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - rightExpr.add(generateConstant(new Randomly(), type)); - } - return new CnosDBInOperation(leftExpr, rightExpr, Randomly.getBoolean()); - } - - public CnosDBExpression generateExpression(int depth, CnosDBDataType originalType) { - return generateExpressionInternal(depth, originalType); - } - - private CnosDBExpression generateExpressionInternal(int depth, CnosDBDataType dataType) throws AssertionError { - if (allowAggregateFunctions && Randomly.getBoolean()) { - return getAggregate(dataType); - } - - if (Randomly.getBooleanWithRatherLowProbability() || depth > maxDepth) { - // generic expression - if (Randomly.getBoolean() || depth > maxDepth) { - if (Randomly.getBooleanWithRatherLowProbability()) { - return generateConstant(r, dataType); - } else { - if (filterColumns(dataType).isEmpty()) { - return generateConstant(r, dataType); - } else { - return createColumnOfType(dataType); - } - } - } else { - if (Randomly.getBoolean()) { - return generateCastExpression(depth + 1, dataType); - } else { - return generateFunctionWithUnknownResult(depth, dataType); - } - } - } else { - switch (dataType) { - case BOOLEAN: - return generateBooleanExpression(depth); - case INT: - return generateIntExpression(depth); - case UINT: - return generateUIntExpression(depth); - case STRING: - return generateStringExpression(depth); - case DOUBLE: - return generateFloatExpression(depth); - case TIMESTAMP: - return generateTimeStampExpression(depth); - default: - throw new AssertionError(dataType); - } - } - } - - private CnosDBExpression generateStringExpression(int depth) { - StringExpression option; - List validOptions = new ArrayList<>(Arrays.asList(StringExpression.values())); - option = Randomly.fromList(validOptions); - - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.STRING); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.STRING); - case CONCAT: - return generateConcat(depth); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateConcat(int depth) { - CnosDBExpression left = generateExpression(depth + 1, CnosDBDataType.STRING); - CnosDBExpression right = generateExpression(depth + 1); - return new CnosDBConcatOperation(left, right); - } - - private CnosDBExpression generateIntExpression(int depth) { - IntExpression option; - option = Randomly.fromOptions(IntExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.INT); - case UNARY_OPERATION: - CnosDBExpression intExpression = generateExpression(depth + 1, CnosDBDataType.INT); - return new CnosDBPrefixOperation(intExpression, - Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.INT); - case BINARY_ARITHMETIC_EXPRESSION: - return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.INT), - generateExpression(depth + 1, CnosDBDataType.INT), - CnosDBBinaryOperator.getRandom(CnosDBDataType.INT)); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateUIntExpression(int depth) { - UIntExpression option = Randomly.fromOptions(UIntExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.UINT); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.UINT); - case BINARY_ARITHMETIC_EXPRESSION: - return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.UINT), - generateExpression(depth + 1, CnosDBDataType.UINT), - CnosDBBinaryOperator.getRandom(CnosDBDataType.UINT)); - default: - throw new AssertionError(); - } - - } - - private CnosDBExpression generateFloatExpression(int depth) { - FloatExpression option; - option = Randomly.fromOptions(FloatExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.DOUBLE); - case UNARY_OPERATION: - CnosDBExpression floatExpression = generateExpression(depth + 1, CnosDBDataType.DOUBLE); - return new CnosDBPrefixOperation(floatExpression, - Randomly.getBoolean() ? PrefixOperator.UNARY_PLUS : PrefixOperator.UNARY_MINUS); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.DOUBLE); - case BINARY_ARITHMETIC_EXPRESSION: - return new CnosDBBinaryArithmeticOperation(generateExpression(depth + 1, CnosDBDataType.DOUBLE), - generateExpression(depth + 1, CnosDBDataType.DOUBLE), - CnosDBBinaryOperator.getRandom(CnosDBDataType.DOUBLE)); - case CONSTANT: - return generateConstant(r, CnosDBDataType.DOUBLE); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateTimeStampExpression(int depth) { - if (Randomly.getBoolean()) { - return generateConstant(r, CnosDBDataType.TIMESTAMP); - } - TimestampExpression option; - option = Randomly.fromOptions(TimestampExpression.values()); - switch (option) { - case CAST: - return generateCastExpression(depth + 1, CnosDBDataType.TIMESTAMP); - case FUNCTION: - return generateFunction(depth + 1, CnosDBDataType.TIMESTAMP); - default: - throw new AssertionError(); - } - } - - private CnosDBExpression generateCastExpression(int depth, CnosDBDataType castToType) { - CnosDBDataType castFromType = Randomly.fromList(CnosDBCastOperation.canCastTo(castToType)); - return new CnosDBCastOperation(generateExpression(depth + 1, castFromType), getCompoundDataType(castToType)); - } - - private CnosDBExpression createColumnOfType(CnosDBDataType type) { - List columns = filterColumns(type); - if (columns.isEmpty()) { - throw new IgnoreMeException(); - } - CnosDBColumn fromList = Randomly.fromList(columns); - return CnosDBColumnValue.create(fromList); - } - - final List filterColumns(CnosDBDataType type) { - if (columns == null) { - return Collections.emptyList(); - } else { - return columns.stream().filter(c -> c.getType() == type).collect(Collectors.toList()); - } - } - - public List generateExpressions(int nr) { - List expressions = new ArrayList<>(); - for (int i = 0; i < nr; i++) { - expressions.add(generateExpression(0)); - } - return expressions; - } - - public CnosDBExpression generateExpression(CnosDBDataType dataType) { - return generateExpression(0, dataType); - } - - public CnosDBExpression generateHavingClause() { - this.allowAggregateFunctions = true; - CnosDBExpression expression = generateExpression(CnosDBDataType.BOOLEAN); - this.allowAggregateFunctions = false; - return expression; - } - - public CnosDBExpression generateAggregate() { - return getAggregate(CnosDBDataType.getRandomType()); - } - - private CnosDBExpression getAggregate(CnosDBDataType dataType) { - if (dataType == CnosDBDataType.BOOLEAN) { - List aggregates = CnosDBAggregateFunction.getAggregates(CnosDBDataType.INT); - CnosDBAggregateFunction agg = Randomly.fromList(aggregates); - return new CnosDBCastOperation(generateArgsForAggregate(dataType, agg), - CnosDBCompoundDataType.create(CnosDBDataType.BOOLEAN)); - } else { - List aggregates = CnosDBAggregateFunction.getAggregates(dataType); - CnosDBAggregateFunction agg = Randomly.fromList(aggregates); - return generateArgsForAggregate(dataType, agg); - } - } - - public CnosDBAggregate generateArgsForAggregate(CnosDBDataType dataType, CnosDBAggregateFunction agg) { - CnosDBDataType[] types = agg.getArgsTypes(dataType); - List args = new ArrayList<>(); - for (CnosDBDataType argType : types) { - args.add(createColumnOfType(argType)); - // args.add(generateExpression(argType)); - } - return new CnosDBAggregate(args, agg); - } - - public CnosDBExpressionGenerator allowAggregates(boolean value) { - allowAggregateFunctions = value; - return this; - } - - @Override - public CnosDBExpression generatePredicate() { - return generateExpression(CnosDBDataType.BOOLEAN); - } - - @Override - public CnosDBExpression negatePredicate(CnosDBExpression predicate) { - return new CnosDBPrefixOperation(predicate, PrefixOperator.NOT); - } - - @Override - public CnosDBExpression isNull(CnosDBExpression expr) { - return new CnosDBPostfixOperation(expr, PostfixOperator.IS_NULL); - } - - private enum BooleanExpression { - POSTFIX_OPERATOR, NOT, BINARY_LOGICAL_OPERATOR, BINARY_COMPARISON, FUNCTION, CAST, LIKE, BETWEEN, IN_OPERATION, - SIMILAR_TO, - } - - private enum StringExpression { - CAST, FUNCTION, CONCAT - } - - private enum IntExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION - } - - private enum UIntExpression { - FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION - } - - private enum FloatExpression { - UNARY_OPERATION, FUNCTION, CAST, BINARY_ARITHMETIC_EXPRESSION, CONSTANT - } - - private enum TimestampExpression { - FUNCTION, CAST - } - -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java deleted file mode 100644 index 0d575d3c7..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBInsertGenerator.java +++ /dev/null @@ -1,59 +0,0 @@ -package sqlancer.cnosdb.gen; - -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.schema.AbstractTableColumn; - -public final class CnosDBInsertGenerator { - - private CnosDBInsertGenerator() { - } - - public static CnosDBOtherQuery insert(CnosDBGlobalState globalState) { - CnosDBTable table = globalState.getSchema().getRandomTable(); - ExpectedErrors errors = new ExpectedErrors(); - errors.add("Column time cannot be null."); - StringBuilder sb = new StringBuilder(); - sb.append("INSERT "); - sb.append(table.getName()); - List columns = table.getRandomNonEmptyColumnSubset(); - sb.append("("); - sb.append(columns.stream().map(AbstractTableColumn::getName).collect(Collectors.joining(", "))); - sb.append(")"); - sb.append(" VALUES"); - - int n = Randomly.smallNumber() + 1; - for (int i = 0; i < n; i++) { - if (i != 0) { - sb.append(", "); - } - insertRow(globalState, sb, columns); - } - - // error - return new CnosDBOtherQuery(sb.toString(), errors); - } - - private static void insertRow(CnosDBGlobalState globalState, StringBuilder sb, List columns) { - sb.append("("); - for (int i = 0; i < columns.size(); i++) { - if (i > 0) { - sb.append(", "); - } - CnosDBExpression generateConstant = CnosDBExpressionGenerator.generateConstant(globalState.getRandomly(), - columns.get(i).getType()); - sb.append(CnosDBVisitor.asString(generateConstant)); - } - sb.append(")"); - } - -} diff --git a/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java b/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java deleted file mode 100644 index c046ad3e9..000000000 --- a/src/sqlancer/cnosdb/gen/CnosDBTableGenerator.java +++ /dev/null @@ -1,77 +0,0 @@ -package sqlancer.cnosdb.gen; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBFieldColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTagColumn; -import sqlancer.cnosdb.query.CnosDBOtherQuery; -import sqlancer.common.query.ExpectedErrors; - -public class CnosDBTableGenerator { - - protected final ExpectedErrors errors = new ExpectedErrors(); - private final String tableName; - private final StringBuilder sb = new StringBuilder(); - private final List columnsToBeAdd = new ArrayList<>(); - private CnosDBTable table; - - public CnosDBTableGenerator(String tableName) { - this.tableName = tableName; - } - - public static CnosDBOtherQuery generate(String tableName) { - return new CnosDBTableGenerator(tableName).generate(); - } - - protected CnosDBOtherQuery generate() { - table = new CnosDBTable(tableName, columnsToBeAdd); - - sb.append("CREATE TABLE"); - if (Randomly.getBoolean()) { - sb.append(" IF NOT EXISTS"); - } - sb.append(" "); - sb.append(tableName); - - sb.append("("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - String name = String.format("f%d", i); - createField(name); - sb.append(", "); - } - - sb.append("TAGS("); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - if (i != 0) { - sb.append(", "); - } - String name = String.format("t%d", i); - createTag(name); - } - sb.append("))"); - return new CnosDBOtherQuery(sb.toString(), new ExpectedErrors()); - } - - private void createField(String name) throws AssertionError { - sb.append(name); - sb.append(" "); - CnosDBDataType type = CnosDBDataType.getRandomTypeWithoutTimeStamp(); - CnosDBCommon.appendDataType(type, sb); - CnosDBFieldColumn c = new CnosDBFieldColumn(name, type); - c.setTable(table); - sb.append(" "); - columnsToBeAdd.add(c); - } - - private void createTag(String name) { - sb.append(name); - CnosDBColumn column = new CnosDBTagColumn(name); - column.setTable(table); - columnsToBeAdd.add(column); - } -} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java deleted file mode 100644 index 472aa8f66..000000000 --- a/src/sqlancer/cnosdb/oracle/CnosDBNoRECBase.java +++ /dev/null @@ -1,23 +0,0 @@ -package sqlancer.cnosdb.oracle; - -import sqlancer.Main; -import sqlancer.MainOptions; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.oracle.TestOracle; - -public abstract class CnosDBNoRECBase implements TestOracle { - protected final CnosDBGlobalState state; - protected final Main.StateLogger logger; - protected final MainOptions options; - protected final CnosDBConnection con; - protected String optimizedQueryString; - protected String unoptimizedQueryString; - - public CnosDBNoRECBase(CnosDBGlobalState state) { - this.state = state; - this.con = state.getConnection(); - this.logger = state.getLogger(); - this.options = state.getOptions(); - } -} diff --git a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java b/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java deleted file mode 100644 index 0c817c655..000000000 --- a/src/sqlancer/cnosdb/oracle/CnosDBNoRECOracle.java +++ /dev/null @@ -1,171 +0,0 @@ -package sqlancer.cnosdb.oracle; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBCompoundDataType; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBCastOperation; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBJoin.CnosDBJoinType; -import sqlancer.cnosdb.ast.CnosDBPostfixText; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.ast.CnosDBSelect.SelectType; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; -import sqlancer.cnosdb.oracle.tlp.CnosDBTLPBase; -import sqlancer.cnosdb.query.CnosDBSelectQuery; -import sqlancer.common.oracle.TestOracle; - -public class CnosDBNoRECOracle extends CnosDBNoRECBase implements TestOracle { - - private final CnosDBSchema s; - - public CnosDBNoRECOracle(CnosDBGlobalState globalState) { - super(globalState); - this.s = globalState.getSchema(); - } - - public static List getJoinStatements(CnosDBGlobalState globalState, List columns, - List tables) { - List joinStatements = new ArrayList<>(); - CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(columns); - for (int i = 1; i < tables.size(); i++) { - CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); - CnosDBTable table = Randomly.fromList(tables); - tables.remove(table); - CnosDBJoinType options = CnosDBJoinType.getRandom(); - CnosDBJoin j = new CnosDBJoin(new CnosDBFromTable(table), joinClause, options); - joinStatements.add(j); - } - // JOIN subqueries - for (int i = 0; i < Randomly.smallNumber(); i++) { - CnosDBTables subqueryTables = globalState.getSchema().getRandomTableNonEmptyTables(); - CnosDBSubquery subquery = CnosDBTLPBase.createSubquery(globalState, String.format("sub%d", i), - subqueryTables); - CnosDBExpression joinClause = gen.generateExpression(CnosDBDataType.BOOLEAN); - CnosDBJoinType options = CnosDBJoinType.getRandom(); - CnosDBJoin j = new CnosDBJoin(subquery, joinClause, options); - joinStatements.add(j); - } - return joinStatements; - } - - @Override - public void check() throws Exception { - CnosDBTables randomTables = s.getRandomTableNonEmptyTables(); - List columns = randomTables.getColumns(); - CnosDBExpression randomWhereCondition = getRandomWhereCondition(columns); - List tables = randomTables.getTables(); - - List joinStatements = getJoinStatements(state, columns, tables); - List fromTables = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); - int secondCount = getUnoptimizedQueryCount(fromTables, randomWhereCondition, joinStatements); - int firstCount = getOptimizedQueryCount(fromTables, List.of(CnosDBColumn.createDummy("f0")), - randomWhereCondition, joinStatements); - if (firstCount == -1 || secondCount == -1) { - throw new IgnoreMeException(); - } - if (firstCount != secondCount) { - String queryFormatString = "-- %s;\n-- count: %d"; - String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, firstCount); - String secondQueryStringWithCount = String.format(queryFormatString, unoptimizedQueryString, secondCount); - state.getState().getLocalState() - .log(String.format("%s\n%s", firstQueryStringWithCount, secondQueryStringWithCount)); - String assertionMessage = String.format("the counts mismatch (%d and %d)!\n%s\n%s", firstCount, secondCount, - firstQueryStringWithCount, secondQueryStringWithCount); - throw new AssertionError(assertionMessage); - } - } - - private CnosDBExpression getRandomWhereCondition(List columns) { - return new CnosDBExpressionGenerator(state).setColumns(columns).generateExpression(CnosDBDataType.BOOLEAN); - } - - private int getUnoptimizedQueryCount(List fromTables, CnosDBExpression randomWhereCondition, - List joinStatements) throws Exception { - CnosDBSelect select = new CnosDBSelect(); - CnosDBCastOperation isTrue = new CnosDBCastOperation(randomWhereCondition, - CnosDBCompoundDataType.create(CnosDBDataType.INT)); - CnosDBPostfixText asText = new CnosDBPostfixText(isTrue, " as count", CnosDBDataType.INT); - select.setFetchColumns(List.of(asText)); - select.setFromList(fromTables); - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int secondCount = 0; - unoptimizedQueryString = "SELECT SUM(count) FROM (" + CnosDBVisitor.asString(select) + ") as res"; - if (options.logEachSelect()) { - logger.writeCurrent(unoptimizedQueryString); - } - CnosDBSelectQuery q = new CnosDBSelectQuery(unoptimizedQueryString, CnosDBExpectedError.expectedErrors()); - CnosDBResultSet rs; - try { - q.executeAndGet(state); - rs = q.getResultSet(); - } catch (Exception e) { - if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - throw new AssertionError(unoptimizedQueryString, e); - } - if (rs == null) { - return -1; - } - - if (rs.next()) { - secondCount += rs.getLong(1); - } - rs.close(); - return secondCount; - } - - private int getOptimizedQueryCount(List randomTables, List columns, - CnosDBExpression randomWhereCondition, List joinStatements) { - CnosDBSelect select = new CnosDBSelect(); - CnosDBColumnValue allColumns = new CnosDBColumnValue(Randomly.fromList(columns)); - select.setFetchColumns(List.of(allColumns)); - select.setFromList(randomTables); - select.setWhereClause(randomWhereCondition); - if (Randomly.getBooleanWithSmallProbability()) { - select.setOrderByClauses(new CnosDBExpressionGenerator(state).setColumns(columns).generateOrderBy()); - } - select.setSelectType(SelectType.ALL); - select.setJoinClauses(joinStatements); - int firstCount = 0; - optimizedQueryString = CnosDBVisitor.asString(select); - if (options.logEachSelect()) { - logger.writeCurrent(optimizedQueryString); - } - CnosDBSelectQuery query = new CnosDBSelectQuery(optimizedQueryString, CnosDBExpectedError.expectedErrors()); - CnosDBResultSet rs; - try { - query.executeAndGet(state); - rs = query.getResultSet(); - while (rs.next()) { - firstCount++; - } - } catch (Exception e) { - if (query.getExpectedErrors().errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - - throw new IgnoreMeException(); - } - return firstCount; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java deleted file mode 100644 index b51624a94..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPAggregateOracle.java +++ /dev/null @@ -1,176 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; - -import sqlancer.ComparatorHelper; -import sqlancer.IgnoreMeException; -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBAggregate; -import sqlancer.cnosdb.ast.CnosDBAggregate.CnosDBAggregateFunction; -import sqlancer.cnosdb.ast.CnosDBAlias; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation; -import sqlancer.cnosdb.ast.CnosDBPostfixOperation.PostfixOperator; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation; -import sqlancer.cnosdb.ast.CnosDBPrefixOperation.PrefixOperator; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.cnosdb.query.CnosDBSelectQuery; -import sqlancer.common.oracle.TestOracle; - -public class CnosDBTLPAggregateOracle extends CnosDBTLPBase implements TestOracle { - - private String firstResult; - private String secondResult; - private String originalQuery; - private String metamorphicQuery; - - public CnosDBTLPAggregateOracle(CnosDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - aggregateCheck(); - } - - protected void aggregateCheck() { - CnosDBAggregateFunction aggregateFunction = Randomly.fromOptions(CnosDBAggregateFunction.MAX, - CnosDBAggregateFunction.MIN, CnosDBAggregateFunction.SUM); - - CnosDBAggregate aggregate = gen.generateArgsForAggregate(aggregateFunction.getRandomReturnType(), - aggregateFunction); - List fetchColumns = new ArrayList<>(); - fetchColumns.add(aggregate); - while (Randomly.getBooleanWithRatherLowProbability()) { - fetchColumns.add(gen.generateAggregate()); - } - select.setFetchColumns(fetchColumns); - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - originalQuery = CnosDBVisitor.asString(select); - firstResult = getAggregateResult(originalQuery); - metamorphicQuery = createMetamorphicUnionQuery(select, aggregate, select.getFromList()); - secondResult = getAggregateResult(metamorphicQuery); - - String queryFormatString = "-- %s;\n-- result: %s"; - String firstQueryString = String.format(queryFormatString, originalQuery, firstResult); - String secondQueryString = String.format(queryFormatString, metamorphicQuery, secondResult); - state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString)); - if (firstResult == null && secondResult != null || firstResult != null && secondResult == null - || firstResult != null && !firstResult.contentEquals(secondResult) - && !ComparatorHelper.isEqualDouble(firstResult, secondResult)) { - if (secondResult != null && secondResult.contains("Inf")) { - throw new IgnoreMeException(); // FIXME: average computation - } - String assertionMessage = String.format("%s: the results mismatch!\n%s\n%s", this.s.getDatabaseName(), - firstQueryString, secondQueryString); - throw new AssertionError(assertionMessage); - } - } - - private String createMetamorphicUnionQuery(CnosDBSelect select, CnosDBAggregate aggregate, - List from) { - String metamorphicQuery; - CnosDBExpression whereClause = gen.generateExpression(CnosDBDataType.BOOLEAN); - CnosDBExpression negatedClause = new CnosDBPrefixOperation(whereClause, PrefixOperator.NOT); - CnosDBExpression notNullClause = new CnosDBPostfixOperation(whereClause, PostfixOperator.IS_NULL); - List mappedAggregate = mapped(aggregate); - CnosDBSelect leftSelect = getSelect(mappedAggregate, from, whereClause, select.getJoinClauses()); - CnosDBSelect middleSelect = getSelect(mappedAggregate, from, negatedClause, select.getJoinClauses()); - CnosDBSelect rightSelect = getSelect(mappedAggregate, from, notNullClause, select.getJoinClauses()); - metamorphicQuery = "SELECT " + getOuterAggregateFunction(aggregate) + " FROM ("; - metamorphicQuery += CnosDBVisitor.asString(leftSelect) + " UNION ALL " + CnosDBVisitor.asString(middleSelect) - + " UNION ALL " + CnosDBVisitor.asString(rightSelect); - metamorphicQuery += ") as asdf"; - return metamorphicQuery; - } - - private String getAggregateResult(String queryString) { - // log TLP Aggregate SELECT queries on the current log file - if (state.getOptions().logEachSelect()) { - // TODO: refactor me - state.getLogger().writeCurrent(queryString); - try { - state.getLogger().getCurrentFileWriter().flush(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - String resultString = null; - - CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, CnosDBExpectedError.expectedErrors()); - try { - q.executeAndGet(state); - CnosDBResultSet result = q.getResultSet(); - - if (result == null || !result.next()) { - throw new IgnoreMeException(); - } - - resultString = result.getString(1); - - } catch (Exception e) { - if (q.getExpectedErrors().errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - } - - return resultString; - } - - private List mapped(CnosDBAggregate aggregate) { - switch (aggregate.getFunction()) { - case SUM: - case MAX: - case MIN: - return aliasArgs(List.of(aggregate)); - // now not support - // case COUNT: - // case AVG: - default: - throw new AssertionError(aggregate.getFunction()); - } - } - - private List aliasArgs(List originalAggregateArgs) { - List args = new ArrayList<>(); - int i = 0; - for (CnosDBExpression expr : originalAggregateArgs) { - args.add(new CnosDBAlias(expr, "agg" + i++)); - } - return args; - } - - private String getOuterAggregateFunction(CnosDBAggregate aggregate) { - if (Objects.requireNonNull(aggregate.getFunction()) == CnosDBAggregateFunction.COUNT) { - return CnosDBAggregateFunction.SUM + "(agg0)"; - } - return aggregate.getFunction() + "(agg0)"; - } - - private CnosDBSelect getSelect(List aggregates, List from, - CnosDBExpression whereClause, List joinList) { - CnosDBSelect leftSelect = new CnosDBSelect(); - leftSelect.setFetchColumns(aggregates); - leftSelect.setFromList(from); - leftSelect.setWhereClause(whereClause); - leftSelect.setJoinClauses(joinList); - if (Randomly.getBooleanWithSmallProbability()) { - leftSelect.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - } - return leftSelect; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java deleted file mode 100644 index bd7ba3b55..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPBase.java +++ /dev/null @@ -1,112 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema; -import sqlancer.cnosdb.CnosDBSchema.CnosDBColumn; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTable; -import sqlancer.cnosdb.CnosDBSchema.CnosDBTables; -import sqlancer.cnosdb.ast.CnosDBColumnValue; -import sqlancer.cnosdb.ast.CnosDBConstant; -import sqlancer.cnosdb.ast.CnosDBExpression; -import sqlancer.cnosdb.ast.CnosDBJoin; -import sqlancer.cnosdb.ast.CnosDBSelect; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBFromTable; -import sqlancer.cnosdb.ast.CnosDBSelect.CnosDBSubquery; -import sqlancer.cnosdb.gen.CnosDBExpressionGenerator; -import sqlancer.cnosdb.oracle.CnosDBNoRECOracle; -import sqlancer.common.gen.ExpressionGenerator; -import sqlancer.common.oracle.TernaryLogicPartitioningOracleBase; -import sqlancer.common.oracle.TestOracle; - -public class CnosDBTLPBase extends TernaryLogicPartitioningOracleBase - implements TestOracle { - - protected CnosDBSchema s; - protected CnosDBTables targetTables; - protected CnosDBExpressionGenerator gen; - protected CnosDBSelect select; - - public CnosDBTLPBase(CnosDBGlobalState state) { - super(state); - } - - public static CnosDBSubquery createSubquery(CnosDBGlobalState globalState, String name, CnosDBTables tables) { - List columns = new ArrayList<>(); - CnosDBExpressionGenerator gen = new CnosDBExpressionGenerator(globalState).setColumns(tables.getColumns()); - for (int i = 0; i < Randomly.smallNumber() + 1; i++) { - columns.add(gen.generateExpression(0)); - } - CnosDBSelect select = new CnosDBSelect(); - select.setFromList(tables.getTables().stream().map(CnosDBFromTable::new).collect(Collectors.toList())); - select.setFetchColumns(columns); - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(0, CnosDBDataType.BOOLEAN)); - } - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - if (Randomly.getBoolean()) { - select.setLimitClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - if (Randomly.getBoolean()) { - select.setOffsetClause(CnosDBConstant.createIntConstant(Randomly.getPositiveOrZeroNonCachedInteger())); - } - } - return new CnosDBSubquery(select, name); - } - - @Override - public void check() throws Exception { - s = state.getSchema(); - targetTables = s.getRandomTableNonEmptyTables(); - List tables = targetTables.getTables(); - List joins = getJoinStatements(targetTables.getColumns(), tables); - generateSelectBase(tables, joins); - } - - protected List getJoinStatements(List columns, List tables) { - return CnosDBNoRECOracle.getJoinStatements(state, columns, tables); - } - - protected void generateSelectBase(List tables, List joins) { - List tableList = tables.stream().map(CnosDBFromTable::new).collect(Collectors.toList()); - gen = new CnosDBExpressionGenerator(state).setColumns(targetTables.getColumns()); - initializeTernaryPredicateVariants(); - select = new CnosDBSelect(); - select.setFetchColumns(generateFetchColumns()); - select.setFromList(tableList); - select.setWhereClause(null); - select.setJoinClauses(joins); - } - - List generateFetchColumns() { - if (Randomly.getBooleanWithRatherLowProbability()) { - return List.of(new CnosDBColumnValue(CnosDBColumn.createDummy("*"))); - } - List fetchColumns = new ArrayList<>(); - List targetColumns = targetTables.getRandomColumnsWithOnlyOneField(); - - ArrayList columns = new ArrayList<>(); - targetColumns.forEach(column -> column.getTable().getColumns().stream() - .filter(field -> field instanceof CnosDBSchema.CnosDBFieldColumn).findFirst().ifPresent(columns::add)); - targetColumns.addAll(columns); - - targetColumns = targetColumns.stream().distinct().collect(Collectors.toList()); - - for (CnosDBColumn c : targetColumns) { - fetchColumns.add(new CnosDBColumnValue(c)); - } - return fetchColumns; - } - - @Override - protected ExpressionGenerator getGen() { - return gen; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java deleted file mode 100644 index 283d59a23..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPHavingOracle.java +++ /dev/null @@ -1,65 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.util.ArrayList; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBComparatorHelper; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType; -import sqlancer.cnosdb.CnosDBVisitor; -import sqlancer.cnosdb.ast.CnosDBExpression; - -public class CnosDBTLPHavingOracle extends CnosDBTLPBase { - - public CnosDBTLPHavingOracle(CnosDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - havingCheck(); - } - - protected void havingCheck() throws Exception { - if (Randomly.getBoolean()) { - select.setWhereClause(gen.generateExpression(CnosDBDataType.BOOLEAN)); - } - select.setGroupByExpressions(gen.generateExpressions(Randomly.smallNumber() + 1)); - select.setHavingClause(null); - String originalQueryString = CnosDBVisitor.asString(select); - List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, - CnosDBExpectedError.expectedErrors(), state); - - boolean orderBy = Randomly.getBoolean(); - if (orderBy) { - select.setOrderByClauses(gen.generateOrderBy()); - } - select.setHavingClause(predicate); - String firstQueryString = CnosDBVisitor.asString(select); - select.setHavingClause(negatedPredicate); - String secondQueryString = CnosDBVisitor.asString(select); - select.setHavingClause(isNullPredicate); - String thirdQueryString = CnosDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, CnosDBExpectedError.expectedErrors()); - CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } - - @Override - protected CnosDBExpression generatePredicate() { - return gen.generateHavingClause(); - } - - @Override - List generateFetchColumns() { - List expressions = gen.allowAggregates(true).generateExpressions(Randomly.smallNumber() + 1); - gen.allowAggregates(false); - return expressions; - } - -} diff --git a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java b/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java deleted file mode 100644 index 8e118435d..000000000 --- a/src/sqlancer/cnosdb/oracle/tlp/CnosDBTLPWhereOracle.java +++ /dev/null @@ -1,46 +0,0 @@ -package sqlancer.cnosdb.oracle.tlp; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import sqlancer.Randomly; -import sqlancer.cnosdb.CnosDBComparatorHelper; -import sqlancer.cnosdb.CnosDBExpectedError; -import sqlancer.cnosdb.CnosDBGlobalState; -import sqlancer.cnosdb.CnosDBVisitor; - -public class CnosDBTLPWhereOracle extends CnosDBTLPBase { - - public CnosDBTLPWhereOracle(CnosDBGlobalState state) { - super(state); - } - - @Override - public void check() throws Exception { - super.check(); - whereCheck(); - } - - protected void whereCheck() throws Exception { - if (Randomly.getBooleanWithRatherLowProbability()) { - select.setOrderByClauses(gen.generateOrderBy()); - } - String originalQueryString = CnosDBVisitor.asString(select); - List resultSet = CnosDBComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, - CnosDBExpectedError.expectedErrors(), state); - - select.setOrderByClauses(Collections.emptyList()); - select.setWhereClause(predicate); - String firstQueryString = CnosDBVisitor.asString(select); - select.setWhereClause(negatedPredicate); - String secondQueryString = CnosDBVisitor.asString(select); - select.setWhereClause(isNullPredicate); - String thirdQueryString = CnosDBVisitor.asString(select); - List combinedString = new ArrayList<>(); - List secondResultSet = CnosDBComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, Randomly.getBoolean(), state, CnosDBExpectedError.expectedErrors()); - CnosDBComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet, originalQueryString, combinedString, - state); - } -} diff --git a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java b/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java deleted file mode 100644 index f0a37056c..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBOtherQuery.java +++ /dev/null @@ -1,32 +0,0 @@ -package sqlancer.cnosdb.query; - -import sqlancer.GlobalState; -import sqlancer.IgnoreMeException; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.query.ExpectedErrors; - -public class CnosDBOtherQuery extends CnosDBQueryAdapter { - private static final long serialVersionUID = 1L; - - public CnosDBOtherQuery(String query, ExpectedErrors errors) { - super(query, errors); - } - - @Override - public boolean couldAffectSchema() { - return true; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - try { - globalState.getConnection().getClient().execute(query); - } catch (Exception e) { - if (this.errors.errorIsExpected(e.getMessage())) { - throw new IgnoreMeException(); - } - } - return true; - } -} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java b/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java deleted file mode 100644 index 115f96ffc..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBQueryAdapter.java +++ /dev/null @@ -1,42 +0,0 @@ -package sqlancer.cnosdb.query; - -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.Query; - -public abstract class CnosDBQueryAdapter extends Query { - private static final long serialVersionUID = 1L; - - String query; - ExpectedErrors errors; - - public CnosDBQueryAdapter(String query, ExpectedErrors errors) { - this.query = query; - this.errors = errors; - } - - @Override - public String getLogString() { - return query; - } - - @Override - public String getQueryString() { - return query; - } - - @Override - public String getUnterminatedQueryString() { - return null; - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public ExpectedErrors getExpectedErrors() { - return errors; - } -} diff --git a/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java b/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java deleted file mode 100644 index dee38abf4..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBQueryProvider.java +++ /dev/null @@ -1,6 +0,0 @@ -package sqlancer.cnosdb.query; - -@FunctionalInterface -public interface CnosDBQueryProvider { - CnosDBOtherQuery getQuery(S globalState) throws Exception; -} diff --git a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java b/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java deleted file mode 100644 index 1c9228182..000000000 --- a/src/sqlancer/cnosdb/query/CnosDBSelectQuery.java +++ /dev/null @@ -1,39 +0,0 @@ -package sqlancer.cnosdb.query; - -import sqlancer.GlobalState; -import sqlancer.cnosdb.client.CnosDBConnection; -import sqlancer.cnosdb.client.CnosDBResultSet; -import sqlancer.common.query.ExpectedErrors; -import sqlancer.common.query.SQLancerResultSet; - -public class CnosDBSelectQuery extends CnosDBQueryAdapter { - private static final long serialVersionUID = 1L; - CnosDBResultSet resultSet; - - public CnosDBSelectQuery(String query, ExpectedErrors errors) { - super(query, errors); - } - - @Override - public boolean couldAffectSchema() { - return false; - } - - @Override - public > boolean execute(G globalState, String... fills) - throws Exception { - globalState.getConnection().getClient().execute(query); - return false; - } - - @Override - public > SQLancerResultSet executeAndGet(G globalState, - String... fills) throws Exception { - resultSet = globalState.getConnection().getClient().executeQuery(query); - return null; - } - - public CnosDBResultSet getResultSet() { - return resultSet; - } -} diff --git a/test/sqlancer/dbms/TestCnosDBNoREC.java b/test/sqlancer/dbms/TestCnosDBNoREC.java deleted file mode 100644 index 1a89a972a..000000000 --- a/test/sqlancer/dbms/TestCnosDBNoREC.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestCnosDBNoREC { - - @Test - public void testCnosDBNoREC() { - assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); - // Run with 0 queries as current implementation is resulting in database crashes - assertEquals(0, - Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", - "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", - "--oracle", "NOREC" })); - } - -} diff --git a/test/sqlancer/dbms/TestCnosDBTLP.java b/test/sqlancer/dbms/TestCnosDBTLP.java deleted file mode 100644 index 4b12aa409..000000000 --- a/test/sqlancer/dbms/TestCnosDBTLP.java +++ /dev/null @@ -1,22 +0,0 @@ -package sqlancer.dbms; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import org.junit.jupiter.api.Test; - -import sqlancer.Main; - -public class TestCnosDBTLP { - - @Test - public void testCnosDBTLP() { - assumeTrue(TestConfig.isEnvironmentTrue(TestConfig.CNOSDB_ENV)); - // Run with 0 queries as current implementation is resulting in database crashes - assertEquals(0, - Main.executeMain(new String[] { "--host", "127.0.0.1", "--port", "8902", "--username", "root", - "--random-seed", "0", "--timeout-seconds", TestConfig.SECONDS, "--num-queries", "0", "cnosdb", - "--oracle", "QUERY_PARTITIONING" })); - } - -} diff --git a/test/sqlancer/dbms/TestConfig.java b/test/sqlancer/dbms/TestConfig.java index f6be45648..f2372f266 100644 --- a/test/sqlancer/dbms/TestConfig.java +++ b/test/sqlancer/dbms/TestConfig.java @@ -5,7 +5,6 @@ public class TestConfig { public static final String SECONDS = "300"; public static final String CLICKHOUSE_ENV = "CLICKHOUSE_AVAILABLE"; - public static final String CNOSDB_ENV = "CNOSDB_AVAILABLE"; public static final String COCKROACHDB_ENV = "COCKROACHDB_AVAILABLE"; public static final String DATABEND_ENV = "DATABEND_AVAILABLE"; public static final String DATAFUSION_ENV = "DATAFUSION_AVAILABLE"; From 7ba8bba56c5397d6b6d345add31ac9be420b4a41 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 23:13:37 +0800 Subject: [PATCH 796/856] Remove Databend bug19738 workaround after upstream fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The issue (databendlabs/databend#19738) was fixed in databendlabs/databend#19740 — re-enable AVG in aggregate testing and stop suppressing the related error messages. Co-Authored-By: Claude Opus 4.7 --- .github/workflows/main.yml | 2 +- src/sqlancer/databend/DatabendBugs.java | 1 - src/sqlancer/databend/DatabendErrors.java | 4 ---- .../tlp/DatabendQueryPartitioningAggregateTester.java | 11 ++++------- 4 files changed, 5 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e85c5c219..437e40350 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -195,7 +195,7 @@ jobs: runs-on: ubuntu-latest services: databend: - image: datafuselabs/databend:v1.2.896-nightly + image: datafuselabs/databend:v1.2.900-nightly env: QUERY_DEFAULT_USER: sqlancer QUERY_DEFAULT_PASSWORD: sqlancer diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index a058ccc32..dd11512d8 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -19,7 +19,6 @@ public final class DatabendBugs { public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 - public static boolean bug19738 = true; // https://github.com/databendlabs/databend/issues/19738 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index 746a4e848..fdd8a3a69 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -47,10 +47,6 @@ public static List getExpressionErrors() { if (DatabendBugs.bug15568) { errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); } - if (DatabendBugs.bug19738) { - errors.add("UnwindError"); - errors.add("unable to cast `NULL`"); - } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE diff --git a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java index 6d52caea3..ee3656413 100644 --- a/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java +++ b/src/sqlancer/databend/test/tlp/DatabendQueryPartitioningAggregateTester.java @@ -10,7 +10,6 @@ import sqlancer.Randomly; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.common.query.SQLancerResultSet; -import sqlancer.databend.DatabendBugs; import sqlancer.databend.DatabendErrors; import sqlancer.databend.DatabendProvider.DatabendGlobalState; import sqlancer.databend.DatabendSchema.DatabendCompositeDataType; @@ -45,12 +44,10 @@ public DatabendQueryPartitioningAggregateTester(DatabendGlobalState state) { @Override public void check() throws SQLException { super.check(); - List aggregateFunctions = new ArrayList<>(List.of(DatabendAggregateFunction.MAX, - DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, DatabendAggregateFunction.COUNT - /* , DatabendAggregateFunction.STDDEV_POP */)); - if (!DatabendBugs.bug19738) { - aggregateFunctions.add(DatabendAggregateFunction.AVG); - } + List aggregateFunctions = new ArrayList<>( + List.of(DatabendAggregateFunction.MAX, DatabendAggregateFunction.MIN, DatabendAggregateFunction.SUM, + DatabendAggregateFunction.COUNT, DatabendAggregateFunction.AVG + /* , DatabendAggregateFunction.STDDEV_POP */)); DatabendAggregateFunction aggregateFunction = Randomly.fromList(aggregateFunctions); DatabendFunctionOperation aggregate = (DatabendAggregateOperation) gen .generateArgsForAggregate(aggregateFunction); From cbd284789221a6ee5680ae3c37753fec9cb815e0 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 23:15:16 +0800 Subject: [PATCH 797/856] Suppress Databend bug19773 (eager-aggregation Decimal mismatch family) Filed as databendlabs/databend#19773. The narrow AVG-only fix in databendlabs/databend#19740 left several related shapes still broken on v1.2.900-nightly, all in the eager-aggregation rewrite path: 1. SUM(decimal_literal) over a cross join inside UNION ALL with an outer aggregate fails with `failed to downcast column Decimal128(...) into ... CoreDecimal`. 2. Plain SUM(decimal_literal) over an N-table cross join fails with `assertion left == right ... Decimal precision: 38 vs 18` once the SUM result is wide enough to be promoted to Decimal128. The original report (#19738) claimed SUM did not trigger the bug; it does, just at higher cardinalities than AVG does. 3. Outer SUM over UNION ALL of inner COUNTs fails with `unable to cast `NULL` to type `UInt64` ... CAST(_eager_final_count (#N) AS UInt64)`. Suppress all three narrowly via distinctive substrings rather than reinstating the broad `UnwindError` / `unable to cast `NULL`` matches the previous commit removed. Co-Authored-By: Claude Opus 4.7 --- src/sqlancer/databend/DatabendBugs.java | 1 + src/sqlancer/databend/DatabendErrors.java | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/sqlancer/databend/DatabendBugs.java b/src/sqlancer/databend/DatabendBugs.java index dd11512d8..ae2f74a33 100644 --- a/src/sqlancer/databend/DatabendBugs.java +++ b/src/sqlancer/databend/DatabendBugs.java @@ -19,6 +19,7 @@ public final class DatabendBugs { public static boolean bug15569 = true; // https://github.com/datafuselabs/databend/issues/15569 public static boolean bug15570 = true; // https://github.com/datafuselabs/databend/issues/15570 public static boolean bug15572 = true; // https://github.com/datafuselabs/databend/issues/15572 + public static boolean bug19773 = true; // https://github.com/databendlabs/databend/issues/19773 private DatabendBugs() { } diff --git a/src/sqlancer/databend/DatabendErrors.java b/src/sqlancer/databend/DatabendErrors.java index fdd8a3a69..3e056d003 100644 --- a/src/sqlancer/databend/DatabendErrors.java +++ b/src/sqlancer/databend/DatabendErrors.java @@ -47,6 +47,11 @@ public static List getExpressionErrors() { if (DatabendBugs.bug15568) { errors.add("Decimal overflow at line : 723 while evaluating function `to_decimal"); } + if (DatabendBugs.bug19773) { + errors.add("failed to downcast column Decimal128"); + errors.add("Decimal(DecimalSize { precision: 38"); + errors.add("_eager_final_count"); + } /* * TODO column为not null 时，注意default不能为null DROP DATABASE IF EXISTS databend2; CREATE DATABASE databend2; USE From da231dd6c28716ec5af893372be7ab6c8f80e7b1 Mon Sep 17 00:00:00 2001 From: Manuel Rigger Date: Mon, 27 Apr 2026 23:55:28 +0800 Subject: [PATCH 798/856] Citus: skip INHERITS generation until citusdata/citus#8553 is fixed Citus's distributed planner returns wrong results when an inheritance parent is cross-joined with a distributed table inside a LEFT JOIN ... ON FALSE and a WHERE filters on the parent column, which the TLP-WHERE oracle keeps tripping over (~1 in 8 Citus CI runs). Make PostgresTableGenerator.generateInherits() protected so CitusTableGenerator can override it as a no-op while the new CitusBugs.bug8553 flag is set. Co-Authored-By: Claude Opus 4.7 --- src/sqlancer/citus/CitusBugs.java | 3 +++ src/sqlancer/citus/gen/CitusTableGenerator.java | 9 +++++++++ src/sqlancer/postgres/gen/PostgresTableGenerator.java | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/citus/CitusBugs.java b/src/sqlancer/citus/CitusBugs.java index a6f4910e6..1fda3be6a 100644 --- a/src/sqlancer/citus/CitusBugs.java +++ b/src/sqlancer/citus/CitusBugs.java @@ -33,6 +33,9 @@ public final class CitusBugs { // https://github.com/citusdata/citus/issues/6298 public static boolean bug6298 = true; + // https://github.com/citusdata/citus/issues/8553 + public static boolean bug8553 = true; + private CitusBugs() { } diff --git a/src/sqlancer/citus/gen/CitusTableGenerator.java b/src/sqlancer/citus/gen/CitusTableGenerator.java index 86e5d40cf..5d6b8b249 100644 --- a/src/sqlancer/citus/gen/CitusTableGenerator.java +++ b/src/sqlancer/citus/gen/CitusTableGenerator.java @@ -1,5 +1,6 @@ package sqlancer.citus.gen; +import sqlancer.citus.CitusBugs; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; import sqlancer.postgres.PostgresSchema; @@ -20,4 +21,12 @@ public static SQLQueryAdapter generate(String tableName, PostgresSchema newSchem return new CitusTableGenerator(tableName, newSchema, generateOnlyKnown, globalState).generate(); } + @Override + protected void generateInherits() { + if (CitusBugs.bug8553) { + return; + } + super.generateInherits(); + } + } diff --git a/src/sqlancer/postgres/gen/PostgresTableGenerator.java b/src/sqlancer/postgres/gen/PostgresTableGenerator.java index 29ccfcf2c..9a7bfb032 100644 --- a/src/sqlancer/postgres/gen/PostgresTableGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableGenerator.java @@ -206,7 +206,7 @@ private void generateUsing() { sb.append(globalState.getRandomTableAccessMethod()); } - private void generateInherits() { + protected void generateInherits() { if (Randomly.getBoolean() && !newSchema.getDatabaseTablesWithoutViews().isEmpty()) { sb.append(" INHERITS("); sb.append(newSchema.getDatabaseTablesRandomSubsetNotEmpty().stream().map(t -> t.getName()) From d2b7151a96c3a482a221969ded411d7ef1dd4cfa Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 30 May 2026 16:39:08 +0100 Subject: [PATCH 799/856] Update MySQL CI to MySQL 9.7.0 and JDBC driver to mysql-connector-j 9.7.0 --- .github/workflows/main.yml | 2 +- pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c511fc0a..4543900b7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -444,7 +444,7 @@ jobs: runs-on: ubuntu-latest services: mysql: - image: mysql:8.4 + image: mysql:9.7.0 env: MYSQL_ROOT_PASSWORD: root ports: diff --git a/pom.xml b/pom.xml index 7c9a1106b..c4bc71f82 100644 --- a/pom.xml +++ b/pom.xml @@ -302,9 +302,9 @@ 3.49.1.0 - mysql - mysql-connector-java - 8.0.30 + com.mysql + mysql-connector-j + 9.7.0 org.mariadb.jdbc From 3efe71a4f193fed29a7815ac9d020e1c83f7f6b8 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 30 May 2026 16:46:17 +0100 Subject: [PATCH 800/856] Add 'incorrect FLOAT value' as expected error following update to MySQL 9.7.0 --- src/sqlancer/mysql/MySQLErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 989c8fed6..6843a27c3 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -49,6 +49,7 @@ public static List getInsertUpdateErrors() { errors.add("doesn't have a default value"); errors.add("Data truncation"); errors.add("Incorrect integer value"); + errors.add("Incorrect FLOAT value"); errors.add("Duplicate entry"); errors.add("Data truncated for column"); errors.add("Data truncated for functional index"); From 725d31de0e2702ecf7d09a6d4e3417d57493f849 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 30 May 2026 16:49:36 +0100 Subject: [PATCH 801/856] Add 'incorrect DOUBLE value' as expected error following update to MySQL 9.7.0 --- src/sqlancer/mysql/MySQLErrors.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 6843a27c3..cb9ad4f01 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -50,6 +50,7 @@ public static List getInsertUpdateErrors() { errors.add("Data truncation"); errors.add("Incorrect integer value"); errors.add("Incorrect FLOAT value"); + errors.add("Incorrect DOUBLE value"); errors.add("Duplicate entry"); errors.add("Data truncated for column"); errors.add("Data truncated for functional index"); From 9dbefd9269463471c5fd3d6199fc182ce993ccfd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 1 Jun 2026 17:32:24 +0100 Subject: [PATCH 802/856] Implement workaround for MySQL CREATE INDEX on integer column bug --- src/sqlancer/mysql/MySQLBugs.java | 4 +++ src/sqlancer/mysql/ast/MySQLConstant.java | 9 +++++++ .../mysql/gen/MySQLInsertGenerator.java | 27 +++++++++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 8cb8a3391..1f9b7ce2f 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -37,6 +37,10 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=114534 public static boolean bug114534 = true; + // https://bugs.mysql.com/bug.php?id=120711 + // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. + public static boolean bug120711 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index 2e4922f8e..5fb0698b9 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -68,6 +68,11 @@ public MySQLDoubleConstant(double val) { } } + @Override + public double getDouble() { + return val; + } + @Override public String getTextRepresentation() { return String.valueOf(val); @@ -381,6 +386,10 @@ public long getInt() { throw new UnsupportedOperationException(); } + public double getDouble() { + throw new UnsupportedOperationException(); + } + public boolean isSigned() { return false; } diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 86083fd2d..696231a24 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -8,10 +8,14 @@ import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLConstant; public class MySQLInsertGenerator { @@ -84,8 +88,27 @@ private SQLQueryAdapter generateInto() { if (c != 0) { sb.append(", "); } - sb.append(MySQLVisitor.asString(gen.generateConstant())); - + MySQLExpression constExpr; + // Bug workaround: for integer columns, reject numeric values that round to 1. Regenerate until valid. + if (MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { + while (true) { + constExpr = gen.generateConstant(); + boolean reject = false; + if (constExpr instanceof MySQLConstant.MySQLIntConstant) { + long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); + reject = value == 1; + } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { + double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); + reject = value >= 0.5 && value < 1.5; + } + if (!reject) { + break; + } + } + } else { + constExpr = gen.generateConstant(); + } + sb.append(MySQLVisitor.asString(constExpr)); } sb.append(")"); } From 2112337c335b2b01e8d6d50a978e853053e70771 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 1 Jun 2026 11:05:16 +0100 Subject: [PATCH 803/856] Implement workaround for MySQL DECIMAL UNIQUE bug --- src/sqlancer/mysql/MySQLBugs.java | 4 ++++ src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 16 ++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 1f9b7ce2f..9f0b7589c 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -37,6 +37,10 @@ public final class MySQLBugs { // https://bugs.mysql.com/bug.php?id=114534 public static boolean bug114534 = true; + // https://bugs.mysql.com/bug.php?id=120710 + // Inserting a NULL and a value which rounds to 0 into a DECIMAL column causes result set mismatch. + public static boolean bug120710 = true; + // https://bugs.mysql.com/bug.php?id=120711 // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. public static boolean bug120711 = true; diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index 696231a24..dc2b51bf7 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -105,6 +105,22 @@ private SQLQueryAdapter generateInto() { break; } } + // Bug workaround: for decimal columns, reject values that round to 0. Regenerate until valid. + } else if (MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { + while (true) { + constExpr = gen.generateConstant(); + boolean reject = false; + if (constExpr instanceof MySQLConstant.MySQLIntConstant) { + long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); + reject = value == 0; + } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { + double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); + reject = value >= -0.5 && value < 0.5; + } + if (!reject) { + break; + } + } } else { constExpr = gen.generateConstant(); } From f857438c035761c63836971503847ed8212c16b1 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 8 Jun 2026 10:22:30 +0800 Subject: [PATCH 804/856] Make the workaround for MySQL insertion bugs more robust against string generation (which may implicitly cast to undesirable integer/double) --- src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index dc2b51bf7..b57ab5158 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -100,6 +100,8 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= 0.5 && value < 1.5; + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 1 + reject = true; } if (!reject) { break; @@ -116,6 +118,8 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= -0.5 && value < 0.5; + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 0 + reject = true; } if (!reject) { break; From 04b7549e04bdc693367086fda6196ddf5645b20c Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 8 Jun 2026 15:10:18 +0800 Subject: [PATCH 805/856] Fix EXPLAIN format following change of default since previous version of MySQL --- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 98641ab26..8af6923e0 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -253,7 +253,7 @@ public List generateFetchColumns(boolean shouldCreateDummy) { @Override public String generateExplainQuery(MySQLSelect select) { - return "EXPLAIN " + select.asString(); + return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed from TRADITIONAL to TREE, hence TRADITIONAL must now be specified } public MySQLAggregate generateAggregate() { From e004455565157ce630499346c20cbe9a2351b100 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 9 Jun 2026 14:40:27 +0800 Subject: [PATCH 806/856] Refactor workaround logic for MySQL insertion bugs for easier extension --- .../mysql/gen/MySQLInsertGenerator.java | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index b57ab5158..d15d9def1 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -89,11 +89,13 @@ private SQLQueryAdapter generateInto() { sb.append(", "); } MySQLExpression constExpr; - // Bug workaround: for integer columns, reject numeric values that round to 1. Regenerate until valid. - if (MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { - while (true) { - constExpr = gen.generateConstant(); - boolean reject = false; + // loop to regenerate until expression is valid (for bug workarounds) + while (true) { + constExpr = gen.generateConstant(); + boolean reject = false; + + // Bug workaround: for integer columns, reject values that round to 1 + if (!reject && MySQLBugs.bug120711 && columns.get(c).getType() == MySQLDataType.INT) { if (constExpr instanceof MySQLConstant.MySQLIntConstant) { long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); reject = value == 1; @@ -103,15 +105,10 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 1 reject = true; } - if (!reject) { - break; - } } - // Bug workaround: for decimal columns, reject values that round to 0. Regenerate until valid. - } else if (MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { - while (true) { - constExpr = gen.generateConstant(); - boolean reject = false; + + // Bug workaround: for decimal columns, reject values that round to 0 + if (!reject && MySQLBugs.bug120710 && columns.get(c).getType() == MySQLDataType.DECIMAL) { if (constExpr instanceof MySQLConstant.MySQLIntConstant) { long value = ((MySQLConstant.MySQLIntConstant) constExpr).getInt(); reject = value == 0; @@ -121,12 +118,11 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 0 reject = true; } - if (!reject) { - break; - } } - } else { - constExpr = gen.generateConstant(); + + if (!reject) { + break; + } } sb.append(MySQLVisitor.asString(constExpr)); } From 383d79c8de0f72fc49ce59a5276ea1a023827561 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 9 Jun 2026 15:21:24 +0800 Subject: [PATCH 807/856] Implement workaround for MySQL CREATE INDEX between NULL inserts CERT bug --- src/sqlancer/mysql/MySQLBugs.java | 4 ++++ src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 9f0b7589c..8032d78ab 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -45,6 +45,10 @@ public final class MySQLBugs { // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. public static boolean bug120711 = true; + // https://bugs.mysql.com/bug.php?id=120712 + // Creating an index in between two NULL inserts causes inconsistent CERT result. + public static boolean bug120712 = true; + private MySQLBugs() { } diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index d15d9def1..a0f44ac9e 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -10,6 +10,7 @@ import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; @@ -120,6 +121,13 @@ private SQLQueryAdapter generateInto() { } } + // Bug workaround: if using CERT oracle, reject NULL values + if (!reject && MySQLBugs.bug120712 && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.CERT)) { + if (constExpr instanceof MySQLConstant.MySQLNullConstant) { + reject = true; + } + } + if (!reject) { break; } From 2eced5c4c33b321105d8d9c17ff2069228f1399e Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 12:48:38 +0800 Subject: [PATCH 808/856] Implement workaround for oracles reporting inconsistency with ZEROFILL in MySQL despite the behaviour being expected --- src/sqlancer/mysql/MySQLBugs.java | 6 +----- src/sqlancer/mysql/gen/MySQLExpressionGenerator.java | 2 +- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 5 +++-- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 8032d78ab..5ad2cfcbd 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -3,12 +3,8 @@ // do not make the fields final to avoid warnings public final class MySQLBugs { - // https://bugs.mysql.com/bug.php?id=99127 0.9 > t0.c0 malfunctions when c0 is - // an INT UNSIGNED - public static boolean bug99127 = true; - // https://bugs.mysql.com/99182 BETWEEN malfunctions for DECIMAL and TEXT - public static boolean bug99181 = true; + public static boolean bug99182 = true; // https://bugs.mysql.com/bug.php?id=99183 public static boolean bug99183 = true; diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 8af6923e0..513867e39 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -112,7 +112,7 @@ public MySQLExpression generateExpression(int depth) { case EXISTS: return getExists(); case BETWEEN_OPERATOR: - if (MySQLBugs.bug99181) { + if (MySQLBugs.bug99182) { // TODO: there are a number of bugs that are triggered by the BETWEEN operator throw new IgnoreMeException(); } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index bc0533295..17e5e9f13 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -13,6 +13,7 @@ import sqlancer.common.query.SQLQueryAdapter; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable.MySQLEngine; @@ -356,10 +357,10 @@ private void appendType(MySQLDataType randomType) { throw new AssertionError(); } if (randomType.isNumeric()) { - if (Randomly.getBoolean() && randomType != MySQLDataType.INT && !MySQLBugs.bug99127) { + if (Randomly.getBoolean() && randomType != MySQLDataType.INT) { sb.append(" UNSIGNED"); } - if (!globalState.usesPQS() && Randomly.getBoolean()) { + if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS || o == MySQLOracleFactory.DQP)) { sb.append(" ZEROFILL"); } } From 4fad414a842a346b788add59608c0a523ef1a737 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 18:24:33 +0800 Subject: [PATCH 809/856] Add loop counter for MySQLInsertGenerator regeneration attempts --- src/sqlancer/mysql/gen/MySQLInsertGenerator.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index a0f44ac9e..a35549405 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -24,6 +24,7 @@ public class MySQLInsertGenerator { private final StringBuilder sb = new StringBuilder(); private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; + private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug workarounds) public MySQLInsertGenerator(MySQLGlobalState globalState, MySQLTable table) { this.globalState = globalState; @@ -91,7 +92,12 @@ private SQLQueryAdapter generateInto() { } MySQLExpression constExpr; // loop to regenerate until expression is valid (for bug workarounds) + int regenerationAttempts = 0; while (true) { + regenerationAttempts++; + if (regenerationAttempts > MAX_REGENERATION_ATTEMPTS) { + throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + " attempts while generating constant for column " + columns.get(c).getName()); + } constExpr = gen.generateConstant(); boolean reject = false; From 5004bc5730e72a9f380de73c4437d3d7dcf76439 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 18:36:32 +0800 Subject: [PATCH 810/856] Format to pass CI tests --- src/sqlancer/mysql/MySQLBugs.java | 3 ++- .../mysql/gen/MySQLExpressionGenerator.java | 4 ++- .../mysql/gen/MySQLInsertGenerator.java | 25 +++++++++++-------- .../mysql/gen/MySQLTableGenerator.java | 4 ++- 4 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/mysql/MySQLBugs.java b/src/sqlancer/mysql/MySQLBugs.java index 5ad2cfcbd..e4fae3cd7 100644 --- a/src/sqlancer/mysql/MySQLBugs.java +++ b/src/sqlancer/mysql/MySQLBugs.java @@ -38,7 +38,8 @@ public final class MySQLBugs { public static boolean bug120710 = true; // https://bugs.mysql.com/bug.php?id=120711 - // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set mismatch. + // Creating an index on an integer-type column, then inserting a value which rounds to 1, causes result set + // mismatch. public static boolean bug120711 = true; // https://bugs.mysql.com/bug.php?id=120712 diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 513867e39..d8ce5dd37 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -253,7 +253,9 @@ public List generateFetchColumns(boolean shouldCreateDummy) { @Override public String generateExplainQuery(MySQLSelect select) { - return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed from TRADITIONAL to TREE, hence TRADITIONAL must now be specified + return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed + // from TRADITIONAL to TREE, hence TRADITIONAL must + // now be specified } public MySQLAggregate generateAggregate() { diff --git a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java index a35549405..0e464dead 100644 --- a/src/sqlancer/mysql/gen/MySQLInsertGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLInsertGenerator.java @@ -7,16 +7,16 @@ import sqlancer.Randomly; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; -import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLBugs; +import sqlancer.mysql.MySQLErrors; import sqlancer.mysql.MySQLGlobalState; import sqlancer.mysql.MySQLOracleFactory; import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLDataType; import sqlancer.mysql.MySQLSchema.MySQLTable; import sqlancer.mysql.MySQLVisitor; -import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExpression; public class MySQLInsertGenerator { @@ -24,7 +24,8 @@ public class MySQLInsertGenerator { private final StringBuilder sb = new StringBuilder(); private final ExpectedErrors errors = new ExpectedErrors(); private final MySQLGlobalState globalState; - private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug workarounds) + private static final int MAX_REGENERATION_ATTEMPTS = 100; // for regenerating expression until valid (for bug + // workarounds) public MySQLInsertGenerator(MySQLGlobalState globalState, MySQLTable table) { this.globalState = globalState; @@ -96,7 +97,8 @@ private SQLQueryAdapter generateInto() { while (true) { regenerationAttempts++; if (regenerationAttempts > MAX_REGENERATION_ATTEMPTS) { - throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + " attempts while generating constant for column " + columns.get(c).getName()); + throw new AssertionError("Exceeded " + MAX_REGENERATION_ATTEMPTS + + " attempts while generating constant for column " + columns.get(c).getName()); } constExpr = gen.generateConstant(); boolean reject = false; @@ -109,7 +111,8 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= 0.5 && value < 1.5; - } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 1 + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may + // be implicitly cast to 1 reject = true; } } @@ -122,16 +125,18 @@ private SQLQueryAdapter generateInto() { } else if (constExpr instanceof MySQLConstant.MySQLDoubleConstant) { double value = ((MySQLConstant.MySQLDoubleConstant) constExpr).getDouble(); reject = value >= -0.5 && value < 0.5; - } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may be implicitly cast to 0 + } else if (constExpr instanceof MySQLConstant.MySQLTextConstant) { // reject strings, which may + // be implicitly cast to 0 reject = true; } } // Bug workaround: if using CERT oracle, reject NULL values - if (!reject && MySQLBugs.bug120712 && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.CERT)) { - if (constExpr instanceof MySQLConstant.MySQLNullConstant) { - reject = true; - } + if (!reject && MySQLBugs.bug120712 + && globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.CERT) + && constExpr instanceof MySQLConstant.MySQLNullConstant) { + reject = true; } if (!reject) { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 17e5e9f13..054a66cb6 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -360,7 +360,9 @@ private void appendType(MySQLDataType randomType) { if (Randomly.getBoolean() && randomType != MySQLDataType.INT) { sb.append(" UNSIGNED"); } - if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS || o == MySQLOracleFactory.DQP)) { + if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS + || o == MySQLOracleFactory.DQP)) { sb.append(" ZEROFILL"); } } From df5f32ecc1f2d52b675d5534ea68880d6439167c Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 17 Jun 2026 20:01:57 +0800 Subject: [PATCH 811/856] Modify MySQLDQEOracle from TEXT to VARCHAR to prevent MEMORY-engine incompatibility --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index 8ddb6f315..bb55e585d 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -467,7 +467,7 @@ private List getErrors() throws SQLException { public void addAuxiliaryColumns(AbstractRelationalTable table) throws SQLException { String tableName = table.getName(); - String addColumnRowID = String.format("ALTER TABLE %s ADD %s TEXT", tableName, COLUMN_ROWID); + String addColumnRowID = String.format("ALTER TABLE %s ADD %s VARCHAR(36)", tableName, COLUMN_ROWID); new SQLQueryAdapter(addColumnRowID).execute(state, false); state.getState().getLocalState().log(addColumnRowID); From f952b256ef9041b2c2b314fa809576625e2f9baa Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 18 Jun 2026 13:49:57 +0800 Subject: [PATCH 812/856] Fix MySQLDQEOracle false positives from non-deterministic ORDER BY LIMIT and known SELECT/DML error discrepancies Appends rowId as an ORDER BY tiebreaker to eliminate non-determinism when user columns contain duplicate values (e.g. NULLs) under LIMIT. Introduces isKnownSelectDMLDiscrepancy to suppress false positives from error codes that MySQL legitimately raises in UPDATE/DELETE but not SELECT (or vice versa) due to differing execution paths: WHERE-clause type coercion (1292, 1366), functional index maintenance (1030, 3751), and range optimizer memory limits (3170). --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 57 ++++++++++++++++++- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index bb55e585d..cefcf4589 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -71,6 +71,10 @@ public String generateSelectStatement(AbstractTables tables, String tableN for (MySQLColumn column : Randomly.nonEmptySubset(mySQLTables.getColumns())) { orderColumns.add(column.getFullQualifiedName()); } + // rowId tiebreaker ensures ORDER BY LIMIT is deterministic when user columns have duplicate values + for (MySQLTable table : mySQLTables.getTables()) { + orderColumns.add(table.getName() + "." + COLUMN_ROWID); + } if (Randomly.getBooleanWithRatherLowProbability()) { generateLimit = true; @@ -185,7 +189,13 @@ public void check() throws SQLException { public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult updateResult) { if (updateResult.hasEmptyErrors()) { if (!selectResult.hasEmptyErrors()) { - return "SELECT has errors, but UPDATE does not."; + // Tolerate SELECT-only discrepancy errors (e.g. 1292 raised in SELECT but not UPDATE + // due to different short-circuit evaluation paths). + boolean selectHasNonDiscrepancyErrors = selectResult.getQueryErrors().stream() + .anyMatch(e -> !isKnownSelectDMLDiscrepancy(e)); + if (selectHasNonDiscrepancyErrors) { + return "SELECT has errors, but UPDATE does not."; + } } if (!selectResult.hasSameAccessedRows(updateResult)) { return "SELECT accessed different rows from UPDATE."; @@ -201,9 +211,14 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult } // update errors should all appear in the select errors + // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in UPDATE but not SELECT + // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); + if (isKnownSelectDMLDiscrepancy(updateError)) { + continue; + } if (!isFound(selectErrors, updateError)) { return "SELECT has different errors from UPDATE."; } @@ -247,7 +262,13 @@ private static boolean isFound(List selectErrors, SQLQueryError t public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult deleteResult) { if (deleteResult.hasEmptyErrors()) { if (!selectResult.hasEmptyErrors()) { - return "SELECT has errors, but DELETE does not."; + // Tolerate SELECT-only discrepancy errors (e.g. 1292 raised in SELECT but not DELETE + // due to different short-circuit evaluation paths). + boolean selectHasNonDiscrepancyErrors = selectResult.getQueryErrors().stream() + .anyMatch(e -> !isKnownSelectDMLDiscrepancy(e)); + if (selectHasNonDiscrepancyErrors) { + return "SELECT has errors, but DELETE does not."; + } } if (!selectResult.hasSameAccessedRows(deleteResult)) { return "SELECT accessed different rows from DELETE."; @@ -263,9 +284,14 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult } // delete errors should all appear in the select errors + // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in DELETE but not SELECT + // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); + if (isKnownSelectDMLDiscrepancy(deleteError)) { + continue; + } if (!isFound(selectErrors, deleteError)) { return "SELECT has different errors from DELETE."; } @@ -349,6 +375,33 @@ private boolean hasDeleteSpecificErrors(SQLQueryResult deleteResult) { } + /* + * Errors that MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. These are + * acceptable discrepancies and should be skipped when checking that DML errors appear in SELECT errors. They are + * not treated as stop errors (hasStopErrors) so row comparison still proceeds normally. + * + * 1292: Truncated incorrect DOUBLE value — WHERE clause type coercion; MySQL may short-circuit in SELECT but + * evaluate fully in UPDATE/DELETE, raising this at ERROR level vs WARNING in SELECT. 1366: Incorrect + * integer/decimal/float value for column — same WHERE clause coercion discrepancy. 1030: Got error from storage + * engine — raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes so cannot + * produce this error. 3170: range_optimizer_max_mem_size exceeded — MySQL applies this memory budget differently + * for SELECT vs DML; the fallback full-scan still evaluates the WHERE predicate correctly. 3751: Data truncated for + * functional index — raised when a functional index expression truncates a value during DML; structurally + * impossible in SELECT. + */ + private static boolean isKnownSelectDMLDiscrepancy(SQLQueryError error) { + switch (error.getCode()) { + case 1030: + case 1292: + case 1366: + case 3170: + case 3751: + return true; + default: + return false; + } + } + private boolean hasStopErrors(SQLQueryResult queryResult) { return queryResult.getQueryErrors().stream() .anyMatch(error -> error.getLevel() == SQLQueryError.ErrorLevel.ERROR); From 2d50787b9f93539d35ccd41cf64aa35f2acdfe58 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 10:10:05 +0800 Subject: [PATCH 813/856] Move MySQLDQEOracle DML discrepancies into enum for maintainability --- src/sqlancer/mysql/oracle/MySQLDQEOracle.java | 57 ++++++++++--------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java index cefcf4589..429ef3c89 100644 --- a/src/sqlancer/mysql/oracle/MySQLDQEOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLDQEOracle.java @@ -211,8 +211,7 @@ public String compareSelectAndUpdate(SQLQueryResult selectResult, SQLQueryResult } // update errors should all appear in the select errors - // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in UPDATE but not SELECT - // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. + // known SELECT/DML discrepancy errors are skipped: see KnownSelectDMLDiscrepancy for the full list. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < updateResult.getQueryErrors().size(); i++) { SQLQueryError updateError = updateResult.getQueryErrors().get(i); @@ -284,8 +283,7 @@ public String compareSelectAndDelete(SQLQueryResult selectResult, SQLQueryResult } // delete errors should all appear in the select errors - // WHERE coercion errors (1292, 1366) are skipped: MySQL may raise these in DELETE but not SELECT - // due to differing short-circuit evaluation of type-incompatible literals in the WHERE clause. + // known SELECT/DML discrepancy errors are skipped: see KnownSelectDMLDiscrepancy for the full list. List selectErrors = new ArrayList<>(selectResult.getQueryErrors()); for (int i = 0; i < deleteResult.getQueryErrors().size(); i++) { SQLQueryError deleteError = deleteResult.getQueryErrors().get(i); @@ -375,31 +373,36 @@ private boolean hasDeleteSpecificErrors(SQLQueryResult deleteResult) { } - /* - * Errors that MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. These are - * acceptable discrepancies and should be skipped when checking that DML errors appear in SELECT errors. They are - * not treated as stop errors (hasStopErrors) so row comparison still proceeds normally. - * - * 1292: Truncated incorrect DOUBLE value — WHERE clause type coercion; MySQL may short-circuit in SELECT but - * evaluate fully in UPDATE/DELETE, raising this at ERROR level vs WARNING in SELECT. 1366: Incorrect - * integer/decimal/float value for column — same WHERE clause coercion discrepancy. 1030: Got error from storage - * engine — raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes so cannot - * produce this error. 3170: range_optimizer_max_mem_size exceeded — MySQL applies this memory budget differently - * for SELECT vs DML; the fallback full-scan still evaluates the WHERE predicate correctly. 3751: Data truncated for - * functional index — raised when a functional index expression truncates a value during DML; structurally - * impossible in SELECT. - */ + // Errors MySQL may raise in UPDATE/DELETE but not SELECT due to different execution paths. Acceptable + // discrepancies that should be skipped; not treated as stop errors so row comparison proceeds normally. + private enum KnownSelectDMLDiscrepancy { + // WHERE clause type coercion: MySQL may short-circuit in SELECT but evaluate fully in UPDATE/DELETE, + // raising this at ERROR level vs WARNING in SELECT. + TRUNCATED_DOUBLE_VALUE(1292), + // Same WHERE clause coercion discrepancy as TRUNCATED_DOUBLE_VALUE. + INCORRECT_COLUMN_VALUE(1366), + // Raised during functional index maintenance on UPDATE/DELETE; SELECT never writes indexes. + STORAGE_ENGINE_ERROR(1030), + // MySQL applies this memory budget differently for SELECT vs DML; the fallback full-scan still + // evaluates the WHERE predicate correctly. + RANGE_OPTIMIZER_MEM_EXCEEDED(3170), + // Raised when a functional index expression truncates a value during DML; structurally impossible in SELECT. + FUNCTIONAL_INDEX_DATA_TRUNCATED(3751); + + private final int code; + + KnownSelectDMLDiscrepancy(int code) { + this.code = code; + } + } + private static boolean isKnownSelectDMLDiscrepancy(SQLQueryError error) { - switch (error.getCode()) { - case 1030: - case 1292: - case 1366: - case 3170: - case 3751: - return true; - default: - return false; + for (KnownSelectDMLDiscrepancy discrepancy : KnownSelectDMLDiscrepancy.values()) { + if (discrepancy.code == error.getCode()) { + return true; + } } + return false; } private boolean hasStopErrors(SQLQueryResult queryResult) { From 9efa585c35fe25594a0bf1f4157974e775925849 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 10:47:47 +0800 Subject: [PATCH 814/856] Ignore expected prefix-key-on-partitioned-table error in MySQLIndexGenerator MySQL rejects CREATE INDEX with a prefix key part (e.g. c0(3)) on a column that participates in PARTITION BY KEY(). MySQLIndexGenerator generated these without checking partition membership and did not include this error in ExpectedErrors, causing checkException to escalate it to a fatal AssertionError. Adds the error substring to the expected errors list. --- src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java index 550893db5..028886831 100644 --- a/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java +++ b/src/sqlancer/mysql/gen/datadef/MySQLIndexGenerator.java @@ -120,6 +120,8 @@ public SQLQueryAdapter create() { errors.add("Data truncated for functional index"); errors.add("used in key specification without a key length"); errors.add("Row size too large"); // seems to happen together with MIN_ROWS in the table declaration + errors.add("in the PARTITION BY KEY() clause is not supported"); // prefix key parts disallowed on + // KEY-partitioned columns return new SQLQueryAdapter(string, errors, true); } From 8fd48d3d3d092485a0c730887a083f925cc4c2ca Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 13:05:14 +0800 Subject: [PATCH 815/856] Fix YugabyteDB CI by adding ysql_yb_enable_listen_notify flag so that LISTEN and NOTIFY statements are enabled --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6c511fc0a..91d19197f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -646,7 +646,7 @@ jobs: - name: Set up Yugabyte run: | docker pull yugabytedb/yugabyte:latest - docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false + docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false --tserver_flags="ysql_yb_enable_listen_notify=true" --master_flags="ysql_yb_enable_listen_notify=true" until pg_isready -h localhost -p 5433; do sleep 1; done - name: Run Tests run: | From 376b9d4c1f6b3118a4926af35e35d163ce73beed Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 19 Jun 2026 13:34:22 +0800 Subject: [PATCH 816/856] Add YCQL readiness check for YugabyteDB so that TestYCQL does not run prematurely --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 91d19197f..02507d58f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -648,6 +648,7 @@ jobs: docker pull yugabytedb/yugabyte:latest docker run -d --name yugabyte -p7000:7000 -p9000:9000 -p5433:5433 -p9042:9042 yugabytedb/yugabyte:latest bin/yugabyted start --daemon=false --tserver_flags="ysql_yb_enable_listen_notify=true" --master_flags="ysql_yb_enable_listen_notify=true" until pg_isready -h localhost -p 5433; do sleep 1; done + until nc -z localhost 9042; do sleep 1; done - name: Run Tests run: | YUGABYTE_AVAILABLE=true mvn -Djacoco.skip=true -Dtest=TestYSQLNoREC test From 2821cbd02890185a210739c42028356b4aa41222 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 11 Jul 2026 20:01:22 +0800 Subject: [PATCH 817/856] Remove NoRECBase and DocumentRemovalOracleBase, as these were dead code --- .../oracle/DocumentRemovalOracleBase.java | 29 ------------------- src/sqlancer/common/oracle/NoRECBase.java | 26 ----------------- 2 files changed, 55 deletions(-) delete mode 100644 src/sqlancer/common/oracle/DocumentRemovalOracleBase.java delete mode 100644 src/sqlancer/common/oracle/NoRECBase.java diff --git a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java b/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java deleted file mode 100644 index b6c0ee509..000000000 --- a/src/sqlancer/common/oracle/DocumentRemovalOracleBase.java +++ /dev/null @@ -1,29 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.GlobalState; -import sqlancer.common.gen.ExpressionGenerator; - -public abstract class DocumentRemovalOracleBase> implements TestOracle { - - protected E predicate; - - protected final S state; - - protected DocumentRemovalOracleBase(S state) { - this.state = state; - } - - protected void initializeDocumentRemovalOracle() { - ExpressionGenerator gen = getGen(); - if (gen == null) { - throw new IllegalStateException(); - } - predicate = gen.generatePredicate(); - if (predicate == null) { - throw new IllegalStateException(); - } - } - - protected abstract ExpressionGenerator getGen(); - -} diff --git a/src/sqlancer/common/oracle/NoRECBase.java b/src/sqlancer/common/oracle/NoRECBase.java deleted file mode 100644 index 2ac0dbb43..000000000 --- a/src/sqlancer/common/oracle/NoRECBase.java +++ /dev/null @@ -1,26 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.Main.StateLogger; -import sqlancer.MainOptions; -import sqlancer.SQLConnection; -import sqlancer.SQLGlobalState; -import sqlancer.common.query.ExpectedErrors; - -public abstract class NoRECBase> implements TestOracle { - - protected final S state; - protected final ExpectedErrors errors = new ExpectedErrors(); - protected final StateLogger logger; - protected final MainOptions options; - protected final SQLConnection con; - protected String optimizedQueryString; - protected String unoptimizedQueryString; - - protected NoRECBase(S state) { - this.state = state; - this.con = state.getConnection(); - this.logger = state.getLogger(); - this.options = state.getOptions(); - } - -} From f984df4cece25768c588774e992f36681bd48655 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 13 Jul 2026 18:30:37 +0800 Subject: [PATCH 818/856] Fix TLPWhereReproducer bug that prevented reducer from performing any reduction --- src/sqlancer/MainOptions.java | 4 ++-- src/sqlancer/common/oracle/TLPWhereOracle.java | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/MainOptions.java b/src/sqlancer/MainOptions.java index a5142fcf0..25b769312 100644 --- a/src/sqlancer/MainOptions.java +++ b/src/sqlancer/MainOptions.java @@ -126,10 +126,10 @@ public class MainOptions { @Parameter(names = "--serialize-reproduce-state", description = "Serialize the state to reproduce") private boolean serializeReproduceState = false; // NOPMD - @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer") + @Parameter(names = "--use-reducer", description = "EXPERIMENTAL Attempt to reduce queries using a simple reducer. Implemented for TLP WHERE and NoREC only") private boolean useReducer = false; // NOPMD - @Parameter(names = "--reduce-ast", description = "EXPERIMENTAL perform AST reduction after statement reduction") + @Parameter(names = "--reduce-ast", description = "EXPERIMENTAL Perform AST reduction after statement reduction") private boolean reduceAST = false; // NOPMD @Parameter(names = "--statement-reducer-max-steps", description = "EXPERIMENTAL Maximum steps the statement reducer will do") diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 14834a62f..31ecc4481 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -106,15 +106,15 @@ public void check() throws SQLException { select.setWhereClause(predicates.isNullPredicate); String thirdQueryString = select.asString(); + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, + firstResultSet, orderBy); + List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); - - reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - firstResultSet, orderBy); } @Override From 73157498bda4fd958e8ac829f39cb4560b2e1284 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 13 Jul 2026 19:06:42 +0800 Subject: [PATCH 819/856] Fix bug where reducer logs were being overwritten, causing logs to end up empty --- src/sqlancer/Main.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index f273f5b95..33cf5fbe3 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -502,11 +502,13 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } - try { - logger.getReduceFileWriter().close(); - logger.reduceFileWriter = null; - } catch (IOException e) { - throw new AssertionError(e); + if (logger.reduceFileWriter != null) { + try { + logger.reduceFileWriter.close(); + logger.reduceFileWriter = null; + } catch (IOException e) { + throw new AssertionError(e); + } } throw new AssertionError("Found a potential bug, please check reducer log for detail."); From 2b3b28b051e75dceb919daf12638b0a0dc817ffe Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 14 Jul 2026 13:46:43 +0800 Subject: [PATCH 820/856] Fix bug where reducer always reduced to single statement --- src/sqlancer/common/oracle/NoRECOracle.java | 16 ++++++++-- .../common/oracle/TLPWhereOracle.java | 29 ++++++++++++------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index caf3dff87..0f043a07f 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -1,7 +1,6 @@ package sqlancer.common.oracle; import java.sql.SQLException; -import java.util.Objects; import java.util.function.Function; import sqlancer.IgnoreMeException; @@ -42,7 +41,20 @@ private static class NoRECReproducer> implements @Override public boolean bugStillTriggers(G globalState) { - return !Objects.equals(optimizedQuery.apply(globalState), unoptimizedQuery.apply(globalState)); + int optimizedCount; + int unoptimizedCount; + try { + optimizedCount = optimizedQuery.apply(globalState); + unoptimizedCount = unoptimizedQuery.apply(globalState); + } catch (RuntimeException | AssertionError e) { + // the queries could not be executed on the reduced database (e.g., a statement they + // depend on was removed), which is not the count mismatch that is being reduced + return false; + } + if (optimizedCount == -1 || unoptimizedCount == -1) { + return false; + } + return optimizedCount != unoptimizedCount; } } diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 31ecc4481..60b8d0c29 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -34,30 +34,37 @@ private class TLPWhereReproducer implements Reproducer { final String secondQueryString; final String thirdQueryString; final String originalQueryString; - final List resultSet; final boolean orderBy; TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, - String originalQueryString, List resultSet, boolean orderBy) { + String originalQueryString, boolean orderBy) { this.firstQueryString = firstQueryString; this.secondQueryString = secondQueryString; this.thirdQueryString = thirdQueryString; this.originalQueryString = originalQueryString; - this.resultSet = resultSet; this.orderBy = orderBy; } @Override public boolean bugStillTriggers(G globalState) { + List firstResultSet; + List combinedString = new ArrayList<>(); + List secondResultSet; try { - List combinedString1 = new ArrayList<>(); - List secondResultSet1 = ComparatorHelper.getCombinedResultSet(firstQueryString, - secondQueryString, thirdQueryString, combinedString1, !orderBy, globalState, errors); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, secondResultSet1, originalQueryString, - combinedString1, globalState); - } catch (AssertionError triggeredError) { + firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + globalState); + secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, globalState, errors); + } catch (SQLException | RuntimeException | AssertionError e) { + // the queries could not be executed on the reduced database (e.g., a statement they + // depend on was removed), which is not the result set mismatch that is being reduced + return false; + } + try { + ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, + combinedString, globalState); + } catch (AssertionError resultSetMismatch) { return true; - } catch (SQLException ignored) { } return false; } @@ -107,7 +114,7 @@ public void check() throws SQLException { String thirdQueryString = select.asString(); reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - firstResultSet, orderBy); + orderBy); List combinedString = new ArrayList<>(); List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, From 7d11ca0bb1b76446c18751cd46058b2c9c801ce8 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 14 Jul 2026 13:56:38 +0800 Subject: [PATCH 821/856] Extend reducer to allow reducing DBMS errors (i.e. errors other than oracle logic bugs) --- src/sqlancer/Main.java | 14 +++- src/sqlancer/Reproducer.java | 10 +++ src/sqlancer/common/oracle/NoRECOracle.java | 67 ++++++++++++---- .../common/oracle/TLPWhereOracle.java | 76 ++++++++++++++++--- .../common/oracle/TestOracleUtils.java | 23 ++++++ 5 files changed, 162 insertions(+), 28 deletions(-) diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index 33cf5fbe3..d65649049 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -502,10 +502,18 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } - if (logger.reduceFileWriter != null) { + String bugInformation = reproducer.getBugInformation(); + if (bugInformation != null) { + // log through newGlobalState's logger: it already holds the reduce file + // writer, and opening it through another StateLogger truncates the file + newGlobalState.getLogger().logReducer(bugInformation); + } + + StateLogger reduceLogger = newGlobalState.getLogger(); + if (reduceLogger.reduceFileWriter != null) { try { - logger.reduceFileWriter.close(); - logger.reduceFileWriter = null; + reduceLogger.reduceFileWriter.close(); + reduceLogger.reduceFileWriter = null; } catch (IOException e) { throw new AssertionError(e); } diff --git a/src/sqlancer/Reproducer.java b/src/sqlancer/Reproducer.java index ef64bd0fe..460cc810d 100644 --- a/src/sqlancer/Reproducer.java +++ b/src/sqlancer/Reproducer.java @@ -2,4 +2,14 @@ public interface Reproducer> { boolean bugStillTriggers(G globalState); + + /** + * Describes how to trigger the bug on the database set up by the reduced statements (e.g., the oracle queries to + * run and the failure to expect), so that the reduced test case is complete without the reproducer object. + * + * @return a human-readable description, or null if the reproducer does not provide one + */ + default String getBugInformation() { + return null; + } } diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 0f043a07f..35f0c26af 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -33,10 +33,19 @@ public class NoRECOracle, J extends Join, private static class NoRECReproducer> implements Reproducer { private final Function optimizedQuery; private final Function unoptimizedQuery; - - NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery) { + private final String optimizedQueryString; + private final String unoptimizedQueryString; + // null if the original bug is a count mismatch; otherwise, the message of the unexpected + // DBMS error that the original queries triggered + private final String expectedErrorMessage; + + NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, + String optimizedQueryString, String unoptimizedQueryString, String expectedErrorMessage) { this.optimizedQuery = optimizedQuery; this.unoptimizedQuery = unoptimizedQuery; + this.optimizedQueryString = optimizedQueryString; + this.unoptimizedQueryString = unoptimizedQueryString; + this.expectedErrorMessage = expectedErrorMessage; } @Override @@ -46,9 +55,16 @@ public boolean bugStillTriggers(G globalState) { try { optimizedCount = optimizedQuery.apply(globalState); unoptimizedCount = unoptimizedQuery.apply(globalState); - } catch (RuntimeException | AssertionError e) { - // the queries could not be executed on the reduced database (e.g., a statement they - // depend on was removed), which is not the count mismatch that is being reduced + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs return false; } if (optimizedCount == -1 || unoptimizedCount == -1) { @@ -56,6 +72,22 @@ public boolean bugStillTriggers(G globalState) { } return optimizedCount != unoptimizedCount; } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the row counts of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); + sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + return sb.toString(); + } } public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { @@ -94,21 +126,28 @@ public void check() throws SQLException { state.getLogger().writeCurrent(unoptimizedQueryString); } - int optimizedCount = shouldUseAggregate ? extractCounts(optimizedQueryString, errors, state) - : countRows(optimizedQueryString, errors, state); - int unoptimizedCount = extractCounts(unoptimizedQueryString, errors, state); + Function optimizedQuery = state -> shouldUseAggregate + ? extractCounts(optimizedQueryString, errors, state) : countRows(optimizedQueryString, errors, state); + Function unoptimizedQuery = state -> extractCounts(unoptimizedQueryString, errors, state); + + int optimizedCount; + int unoptimizedCount; + try { + optimizedCount = optimizedQuery.apply(state); + unoptimizedCount = unoptimizedQuery.apply(state); + } catch (AssertionError unexpectedError) { + reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, + unoptimizedQueryString, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } if (optimizedCount == -1 || unoptimizedCount == -1) { throw new IgnoreMeException(); } if (unoptimizedCount != optimizedCount) { - Function optimizedQuery = state -> shouldUseAggregate - ? extractCounts(optimizedQueryString, errors, state) - : countRows(optimizedQueryString, errors, state); - - Function unoptimizedQuery = state -> extractCounts(unoptimizedQueryString, errors, state); - reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery); + reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, + unoptimizedQueryString, null); String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 60b8d0c29..968064158 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -35,14 +35,18 @@ private class TLPWhereReproducer implements Reproducer { final String thirdQueryString; final String originalQueryString; final boolean orderBy; + // null if the original bug is a result set mismatch; otherwise, the message of the + // unexpected DBMS error that the original queries triggered + final String expectedErrorMessage; TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, - String originalQueryString, boolean orderBy) { + String originalQueryString, boolean orderBy, String expectedErrorMessage) { this.firstQueryString = firstQueryString; this.secondQueryString = secondQueryString; this.thirdQueryString = thirdQueryString; this.originalQueryString = originalQueryString; this.orderBy = orderBy; + this.expectedErrorMessage = expectedErrorMessage; } @Override @@ -53,11 +57,23 @@ public boolean bugStillTriggers(G globalState) { try { firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + if (firstQueryString == null) { + // the original bug was a DBMS error on the original query alone, which no + // longer occurs + return false; + } secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, globalState, errors); - } catch (SQLException | RuntimeException | AssertionError e) { - // the queries could not be executed on the reduced database (e.g., a statement they - // depend on was removed), which is not the result set mismatch that is being reduced + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs return false; } try { @@ -68,6 +84,32 @@ public boolean bugStillTriggers(G globalState) { } return false; } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (firstQueryString != null) { + if (orderBy) { + sb.append("-- ").append(firstQueryString).append(';').append(System.lineSeparator()); + sb.append("-- ").append(secondQueryString).append(';').append(System.lineSeparator()); + sb.append("-- ").append(thirdQueryString).append(';').append(System.lineSeparator()); + } else { + sb.append("-- ").append(firstQueryString).append(" UNION ALL ").append(secondQueryString) + .append(" UNION ALL ").append(thirdQueryString).append(';') + .append(System.lineSeparator()); + } + } + return sb.toString(); + } } public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { @@ -96,8 +138,14 @@ public void check() throws SQLException { String originalQueryString = select.asString(); generatedQueryString = originalQueryString; - List firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - state); + List firstResultSet; + try { + firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + } catch (AssertionError unexpectedError) { + reproducer = new TLPWhereReproducer(null, null, null, originalQueryString, false, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } boolean orderBy = Randomly.getBooleanWithSmallProbability(); if (orderBy) { @@ -113,13 +161,19 @@ public void check() throws SQLException { select.setWhereClause(predicates.isNullPredicate); String thirdQueryString = select.asString(); - reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - orderBy); - List combinedString = new ArrayList<>(); - List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, state, errors); + List secondResultSet; + try { + secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, state, errors); + } catch (AssertionError unexpectedError) { + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, + originalQueryString, orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, + orderBy, null); ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); } diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java index bab2e26c9..5233cfa80 100644 --- a/src/sqlancer/common/oracle/TestOracleUtils.java +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -34,6 +34,29 @@ public static final class PredicateVariants, C extends A return new AbstractTables<>(Randomly.nonEmptySubset(schema.getDatabaseTables())); } + /** + * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the + * AssertionError that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it + * to check that a reduced test case still triggers the same error, rather than an unrelated one introduced by the + * reduction itself. + * + * @param error + * the AssertionError wrapping the DBMS error + * + * @return the message of the innermost cause that has one, or the error's own message + */ + public static String getUnexpectedErrorMessage(AssertionError error) { + String message = error.getMessage(); + Throwable current = error.getCause(); + while (current != null) { + if (current.getMessage() != null) { + message = current.getMessage(); + } + current = current.getCause(); + } + return message; + } + public static , T extends AbstractTable, C extends AbstractTableColumn> PredicateVariants initializeTernaryPredicateVariants( PartitionGenerator gen, E predicate) { if (gen == null) { From a02b149a65598b8af893717ceacee764f9d6c395 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 14 Jul 2026 14:47:01 +0800 Subject: [PATCH 822/856] Clarify logs when --use-reducer is applied (the log outside of the reduce directory gives the final reduced test case, while the log inside the reduce directory gives the step-by-step reduction) --- src/sqlancer/ASTBasedReducer.java | 6 +- src/sqlancer/Main.java | 76 +++++++++++++------ src/sqlancer/StateToReproduce.java | 4 + src/sqlancer/StatementReducer.java | 3 +- .../common/oracle/TLPWhereOracle.java | 3 +- .../common/oracle/TestOracleUtils.java | 7 +- 6 files changed, 66 insertions(+), 33 deletions(-) diff --git a/src/sqlancer/ASTBasedReducer.java b/src/sqlancer/ASTBasedReducer.java index 876a2da12..f9468af76 100644 --- a/src/sqlancer/ASTBasedReducer.java +++ b/src/sqlancer/ASTBasedReducer.java @@ -103,8 +103,7 @@ public void reduce(G state, Reproducer reproducer, G newGlobalState) throws E }); if (!initFlag) { - newGlobalState.getLogger() - .logReducer("warning: failed parsing the statement at transformer : " + t); + System.out.println("Error when parsing the statement at transformer :" + t); continue; } t.apply(); @@ -114,7 +113,8 @@ public void reduce(G state, Reproducer reproducer, G newGlobalState) throws E } while (observeChange); newGlobalState.getState().setStatements(new ArrayList<>(reducedStatements)); - newGlobalState.getLogger().logReduced(newGlobalState.getState()); + newGlobalState.getLogger().logReduced(newGlobalState.getState(), + "AST-based reduction finished; the following statements remain"); } public boolean bugStillTriggers() throws Exception { diff --git a/src/sqlancer/Main.java b/src/sqlancer/Main.java index d65649049..47ba2aedf 100644 --- a/src/sqlancer/Main.java +++ b/src/sqlancer/Main.java @@ -80,6 +80,9 @@ public static final class StateLogger { private FileWriter queryPlanFileWriter; private FileWriter reduceFileWriter; private Path reproduceFilePath; + private List> reduceSetupStatements; + private String reduceBugInformation; + private int nrReductionAttempts; private static final List INITIALIZED_PROVIDER_NAMES = new ArrayList<>(); private final boolean logEachSelect; @@ -262,33 +265,32 @@ public void writeQueryPlan(String queryPlan) { } } - public void logReducer(String reducerLog) { - FileWriter reduceFileWriter = getReduceFileWriter(); - - StringBuilder sb = new StringBuilder(); - sb.append("[reducer log] "); - sb.append(reducerLog); - try { - reduceFileWriter.write(sb.toString()); - } catch (IOException e) { - throw new AssertionError(e); - } finally { - try { - reduceFileWriter.flush(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } + public void setReductionContext(List> setupStatements, String bugInformation) { + this.reduceSetupStatements = setupStatements; + this.reduceBugInformation = bugInformation; } public void logReduced(StateToReproduce state) { + nrReductionAttempts++; + logReduced(state, "Reduction attempt " + nrReductionAttempts + + ": the bug was still triggered with the following statements"); + } + + public void logReduced(StateToReproduce state, String description) { FileWriter reduceFileWriter = getReduceFileWriter(); StringBuilder sb = new StringBuilder(); - for (Query s : state.getStatements()) { - sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); + sb.append("-- ").append(description).append(System.lineSeparator()); + if (reduceSetupStatements != null && !reduceSetupStatements.isEmpty()) { + appendStatements(sb, reduceSetupStatements); + // e.g. DROP DATABASE IF EXISTS db; CREATE DATABASE db; USE db; + // these statements are executed at the start of every test case and are never reduced + } + appendStatements(sb, state.getStatements()); + if (reduceBugInformation != null) { + sb.append(reduceBugInformation); } + sb.append(System.lineSeparator()); try { reduceFileWriter.write(sb.toString()); @@ -305,6 +307,12 @@ public void logReduced(StateToReproduce state) { } + private void appendStatements(StringBuilder sb, List> statements) { + for (Query s : statements) { + sb.append(databaseProvider.getLoggableFactory().createLoggable(s.getLogString()).getLogString()); + } + } + public void logException(Throwable reduce, StateToReproduce state) { Loggable stackTrace = getStackTrace(reduce); FileWriter logFileWriter2 = getLogFileWriter(); @@ -460,6 +468,9 @@ public void run() throws Exception { if (options.logEachSelect()) { logger.writeCurrent(state.getState()); } + // statements logged so far stem from the database setup (e.g., DROP DATABASE IF + // EXISTS, CREATE DATABASE, USE), performed by createDatabase + int nrSetupStatements = stateToRepro.getStatements().size(); Reproducer reproducer = null; if (options.enableQPG()) { provider.generateAndTestDatabaseWithQueryPlanGuidance(state); @@ -484,6 +495,17 @@ public void run() throws Exception { logger.getReduceFileWriter().write("current oracle does not support experimental reducer."); throw new IgnoreMeException(); } + + // reduce only the generation statements: the database setup (logged by + // createDatabase) is re-executed by the reducers for every candidate, and the + // oracle queries (logged by the oracle's local state) by the reproducer + List> allStatements = new ArrayList<>(stateToRepro.getStatements()); + List> setupStatements = new ArrayList<>(allStatements.subList(0, nrSetupStatements)); + List> oracleQueryStatements = stateToRepro.getLocalState() == null ? new ArrayList<>() + : new ArrayList<>(stateToRepro.getLocalState().getStatements()); + stateToRepro.setStatements(new ArrayList<>(allStatements.subList(nrSetupStatements, + allStatements.size() - oracleQueryStatements.size()))); + G newGlobalState = createGlobalState(); newGlobalState.setState(stateToRepro); newGlobalState.setRandomly(r); @@ -493,6 +515,7 @@ public void run() throws Exception { QueryManager newManager = new QueryManager<>(newGlobalState); newGlobalState.setStateLogger(new StateLogger(databaseName, provider, options)); newGlobalState.setManager(newManager); + newGlobalState.getLogger().setReductionContext(setupStatements, reproducer.getBugInformation()); Reducer reducer = new StatementReducer<>(provider); reducer.reduce(state, reproducer, newGlobalState); @@ -502,11 +525,18 @@ public void run() throws Exception { astBasedReducer.reduce(state, reproducer, newGlobalState); } + // reassemble the statements so that the main log looks like one produced + // without the reducer, with the generation statements replaced by the reduced + // ones and the oracle queries at the end + List> finalStatements = new ArrayList<>(setupStatements); + finalStatements.addAll(stateToRepro.getStatements()); + finalStatements.addAll(oracleQueryStatements); + stateToRepro.setStatements(finalStatements); String bugInformation = reproducer.getBugInformation(); if (bugInformation != null) { - // log through newGlobalState's logger: it already holds the reduce file - // writer, and opening it through another StateLogger truncates the file - newGlobalState.getLogger().logReducer(bugInformation); + for (String line : bugInformation.split(System.lineSeparator())) { + stateToRepro.logStatement(line); + } } StateLogger reduceLogger = newGlobalState.getLogger(); diff --git a/src/sqlancer/StateToReproduce.java b/src/sqlancer/StateToReproduce.java index e44d0ccf6..17bb367fd 100644 --- a/src/sqlancer/StateToReproduce.java +++ b/src/sqlancer/StateToReproduce.java @@ -128,6 +128,10 @@ public void log(String s) { statements.add(databaseProvider.getLoggableFactory().getQueryForStateToReproduce(s)); } + public List> getStatements() { + return Collections.unmodifiableList(statements); + } + @Override public void close() { if (!success) { diff --git a/src/sqlancer/StatementReducer.java b/src/sqlancer/StatementReducer.java index e066aca84..6545fb2af 100644 --- a/src/sqlancer/StatementReducer.java +++ b/src/sqlancer/StatementReducer.java @@ -77,7 +77,8 @@ && hasNotReachedLimit(currentReduceTime, maxReduceTime)) { // System.out.println("Reduced query:"); // printQueries(knownToReproduceBugStatements); newGlobalState.getState().setStatements(new ArrayList<>(knownToReproduceBugStatements)); - newGlobalState.getLogger().logReduced(newGlobalState.getState()); + newGlobalState.getLogger().logReduced(newGlobalState.getState(), + "Statement reduction finished; the following statements remain"); } diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 968064158..8fc87ac3d 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -104,8 +104,7 @@ public String getBugInformation() { sb.append("-- ").append(thirdQueryString).append(';').append(System.lineSeparator()); } else { sb.append("-- ").append(firstQueryString).append(" UNION ALL ").append(secondQueryString) - .append(" UNION ALL ").append(thirdQueryString).append(';') - .append(System.lineSeparator()); + .append(" UNION ALL ").append(thirdQueryString).append(';').append(System.lineSeparator()); } } return sb.toString(); diff --git a/src/sqlancer/common/oracle/TestOracleUtils.java b/src/sqlancer/common/oracle/TestOracleUtils.java index 5233cfa80..9bef86762 100644 --- a/src/sqlancer/common/oracle/TestOracleUtils.java +++ b/src/sqlancer/common/oracle/TestOracleUtils.java @@ -35,10 +35,9 @@ public static final class PredicateVariants, C extends A } /** - * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the - * AssertionError that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it - * to check that a reduced test case still triggers the same error, rather than an unrelated one introduced by the - * reduction itself. + * Extracts the message of the DBMS error that caused an oracle query to fail unexpectedly, from the AssertionError + * that wraps it (see, e.g., ComparatorHelper#getResultSetFirstColumnAsString). Reproducers use it to check that a + * reduced test case still triggers the same error, rather than an unrelated one introduced by the reduction itself. * * @param error * the AssertionError wrapping the DBMS error From abbdeed6cbe4bad3c9c7e43b92c880ab47cedf5a Mon Sep 17 00:00:00 2001 From: splf Date: Mon, 27 Jul 2026 23:03:27 +0500 Subject: [PATCH 823/856] Fix MySQL index hint syntax in the DQP hint generator MySQL's grammar for index-level optimizer hints is hint_name(tbl_name index_name [, index_name] ...): a space separates the table name from the first index name, not a comma. MySQLHintGenerator emitted NO_INDEX(t0, PRIMARY), which MySQL rejects with warning 1064 and silently ignores, leaving the query plan unchanged. This affects the 16 of 32 hint kinds that go through indexesHint(). --- src/sqlancer/mysql/gen/MySQLHintGenerator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLHintGenerator.java b/src/sqlancer/mysql/gen/MySQLHintGenerator.java index 141aea279..dc6138b23 100644 --- a/src/sqlancer/mysql/gen/MySQLHintGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLHintGenerator.java @@ -170,7 +170,7 @@ private void indexesHint(String string) { MySQLTable table = Randomly.fromList(tables); List allIndexes = table.getIndexes(); sb.append(table.getName()); - sb.append(", "); + sb.append(" "); if (allIndexes.isEmpty()) { sb.append("PRIMARY"); } else { From eb5e013a227eae7834c5b1f3865c451e57bb3f51 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 9 Jul 2026 13:21:12 +0800 Subject: [PATCH 824/856] Add EET (Equivalent Expression Transformation) oracle for MySQL This commit implements EET from Jiang & Su, "Detecting Logic Bugs in Database Engines via Equivalent Expression Transformation" (OSDI'24). Common, DBMS-independent core: - EETTransformation: the 7 transformation rules of the paper's Table 2. - EETNodeFactory: primitive node constructors the rules are built from. - EETGenerator: generator interface an EET-capable DBMS implements. - EETOracle: orchestration and result comparison (paper Figure 4), with a Reproducer for confirmation/reduction. MySQL-specific implementation: - MySQLEETTransformer recursively rewrites the MySQL AST, rebuilding each node from its transformed children and threading a boolean/scalar context flag so the determined-boolean rules are only applied where sound. - MySQLEETNodeFactory builds the MySQL nodes; MySQLExpressionGenerator implements EETGenerator; MySQLUnaryPrefixOperation exposes getOp(); MySQLOracleFactory registers the EET oracle. - MySQLTableGenerator skips ZEROFILL for EET (as for TLP_WHERE/PQS/DQP) to avoid display-metadata-only false positives while preserving search space. Runs via: java -jar target/sqlancer-*.jar mysql --oracle EET --- src/sqlancer/common/gen/EETGenerator.java | 44 +++++++ .../common/oracle/EETNodeFactory.java | 39 ++++++ src/sqlancer/common/oracle/EETOracle.java | 121 +++++++++++++++++ .../common/oracle/EETTransformation.java | 83 ++++++++++++ src/sqlancer/mysql/MySQLOracleFactory.java | 10 ++ .../mysql/ast/MySQLUnaryPrefixOperation.java | 4 + .../mysql/gen/MySQLEETNodeFactory.java | 70 ++++++++++ .../mysql/gen/MySQLEETTransformer.java | 124 ++++++++++++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 13 +- .../mysql/gen/MySQLTableGenerator.java | 2 +- 10 files changed, 508 insertions(+), 2 deletions(-) create mode 100644 src/sqlancer/common/gen/EETGenerator.java create mode 100644 src/sqlancer/common/oracle/EETNodeFactory.java create mode 100644 src/sqlancer/common/oracle/EETOracle.java create mode 100644 src/sqlancer/common/oracle/EETTransformation.java create mode 100644 src/sqlancer/mysql/gen/MySQLEETNodeFactory.java create mode 100644 src/sqlancer/mysql/gen/MySQLEETTransformer.java diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java new file mode 100644 index 000000000..f468ff6d2 --- /dev/null +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -0,0 +1,44 @@ +package sqlancer.common.gen; + +import java.util.List; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like + * the other oracle generators), an EET generator can transform an expression into a semantically equivalent one + * according to the EET transformation rules. + */ +public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { + + EETGenerator setTablesAndColumns(AbstractTables tables); + + S generateSelect(); + + List getRandomJoinClauses(); + + List getTableRefs(); + + List generateFetchColumns(boolean shouldCreateDummy); + + E generateBooleanExpression(); + + /** + * Transforms an expression into a semantically equivalent one (the core of EET). Typically this recursively + * traverses the expression's AST and replaces sub-expressions with equivalent ones. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (e.g. a WHERE predicate); this controls + * which transformation rules are applicable + * + * @return a semantically equivalent expression + */ + E transformExpression(E expr, boolean booleanContext); +} diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java new file mode 100644 index 000000000..2cc933ced --- /dev/null +++ b/src/sqlancer/common/oracle/EETNodeFactory.java @@ -0,0 +1,39 @@ +package sqlancer.common.oracle; + +/** + * Factory for constructing the AST nodes needed by the {@link EETTransformation equivalent expression transformation} + * rules. Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface + * lets the (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. + * + * @param + * the DBMS-specific expression type + */ +public interface EETNodeFactory { + + /** Builds {@code left AND right}. */ + E and(E left, E right); + + /** Builds {@code left OR right}. */ + E or(E left, E right); + + /** Builds {@code NOT expr}. */ + E not(E expr); + + /** Builds {@code expr IS NULL}. */ + E isNull(E expr); + + /** Builds {@code expr IS NOT NULL}. */ + E isNotNull(E expr); + + /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ + E caseWhen(E condition, E thenExpr, E elseExpr); + + /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ + E generateBooleanExpression(); + + /** + * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not + * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + */ + boolean isCaseWhenApplicable(E expr); +} diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java new file mode 100644 index 000000000..0f192f9ab --- /dev/null +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -0,0 +1,121 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.ComparatorHelper; +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.ast.newast.Join; +import sqlancer.common.ast.newast.Select; +import sqlancer.common.gen.EETGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * EET (Equivalent Expression Transformation) oracle, based on "Detecting Logic Bugs in Database Engines via Equivalent + * Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * The oracle generates a random query and then transforms its expressions (the WHERE predicate and the fetch columns) + * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation preserves + * semantics, the original and the transformed query must return the same result set; any discrepancy indicates a logic + * bug in the DBMS. + */ +public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private EETGenerator gen; + private final ExpectedErrors errors; + + private Reproducer reproducer; + private String generatedQueryString; + + private final class EETReproducer implements Reproducer { + private final String originalQueryString; + private final String transformedQueryString; + private final List resultSet; + + EETReproducer(String originalQueryString, String transformedQueryString, List resultSet) { + this.originalQueryString = originalQueryString; + this.transformedQueryString = transformedQueryString; + this.resultSet = resultSet; + } + + @Override + public boolean bugStillTriggers(G globalState) { + try { + List transformedResultSet = ComparatorHelper + .getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); + ComparatorHelper.assumeResultSetsAreEqual(resultSet, transformedResultSet, originalQueryString, + List.of(transformedQueryString), globalState); + } catch (AssertionError triggeredError) { + return true; + } catch (SQLException ignored) { + } + return false; + } + } + + public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + reproducer = null; + S schema = state.getSchema(); + AbstractTables targetTables = TestOracleUtils.getRandomTableNonEmptyTables(schema); + gen = gen.setTablesAndColumns(targetTables); + + Z select = gen.generateSelect(); + select.setJoinClauses(gen.getRandomJoinClauses()); + select.setFromList(gen.getTableRefs()); + List fetchColumns = gen.generateFetchColumns(true); + select.setFetchColumns(fetchColumns); + E whereClause = gen.generateBooleanExpression(); + select.setWhereClause(whereClause); + + String originalQueryString = select.asString(); + generatedQueryString = originalQueryString; + List originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + state); + + // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, + // while the WHERE clause is evaluated in a boolean context. + List transformedFetchColumns = fetchColumns.stream().map(c -> gen.transformExpression(c, false)) + .collect(Collectors.toList()); + select.setFetchColumns(transformedFetchColumns); + select.setWhereClause(gen.transformExpression(whereClause, true)); + + String transformedQueryString = select.asString(); + List transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, + errors, state); + + ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, + List.of(transformedQueryString), state); + + reproducer = new EETReproducer(originalQueryString, transformedQueryString, originalResultSet); + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/common/oracle/EETTransformation.java b/src/sqlancer/common/oracle/EETTransformation.java new file mode 100644 index 000000000..ab9b7fefe --- /dev/null +++ b/src/sqlancer/common/oracle/EETTransformation.java @@ -0,0 +1,83 @@ +package sqlancer.common.oracle; + +import sqlancer.Randomly; + +/** + * Implements the semantic-preserving expression transformation rules of EET (Equivalent Expression Transformation, Jiang + * & Su, OSDI'24), Table 2. Given an expression, {@link #applyRandomRule} returns a semantically equivalent + * expression built from the primitives provided by an {@link EETNodeFactory}. The rules are DBMS-independent; only the + * node construction (via the factory) is DBMS-specific. + * + *

+ * The rules rely on two always-determined boolean expressions built from an arbitrary boolean {@code p}: + *

    + *
  • {@code true_expr(p) = p OR (NOT p) OR (p IS NULL)}, which always evaluates to TRUE, and
    • + *
  • {@code false_expr(p) = p AND (NOT p) AND (p IS NOT NULL)}, which always evaluates to FALSE.
    • + *
+ * + * @param + * the DBMS-specific expression type + */ +public class EETTransformation { + + private final EETNodeFactory factory; + + public EETTransformation(EETNodeFactory factory) { + this.factory = factory; + } + + // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE + private E trueExpr() { + E p = factory.generateBooleanExpression(); + return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); + } + + // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE + private E falseExpr() { + E p = factory.generateBooleanExpression(); + return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); + } + + /** + * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen, applicable + * transformation rule. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (e.g. a WHERE predicate or an operand of a + * logical operator). Only in a boolean context may the determined-boolean rules (No. 1 and 2), which + * reduce the expression to a boolean value, be applied; in a scalar context they would change the + * expression's value/type and are therefore excluded. + * + * @return a semantically equivalent expression + */ + public E applyRandomRule(E expr, boolean booleanContext) { + int rule; + if (booleanContext) { + // Rules No. 1-6 are all value-preserving in a boolean context. + rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); + } else { + if (!factory.isCaseWhenApplicable(expr)) { + return expr; // rule No. 7: transform the expression to itself + } + // In a scalar context only the CASE WHEN rules preserve the exact value and type. + rule = Randomly.fromOptions(3, 4, 5, 6); + } + switch (rule) { + case 1: // bool_expr => false_expr OR bool_expr + return factory.or(falseExpr(), expr); + case 2: // bool_expr => true_expr AND bool_expr + return factory.and(trueExpr(), expr); + case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END + return factory.caseWhen(falseExpr(), expr, expr); + case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END + return factory.caseWhen(trueExpr(), expr, expr); + case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END + case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END + return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); + default: + throw new AssertionError(rule); + } + } +} diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index 83e08677a..ed5ddc489 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -5,6 +5,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.EETOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; @@ -82,5 +83,14 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLDQEOracle(globalState); } + }, + EET { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); + return new EETOracle<>(globalState, gen, expectedErrors); + } }; } diff --git a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java index c87048a4b..45a1fe888 100644 --- a/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java +++ b/src/sqlancer/mysql/ast/MySQLUnaryPrefixOperation.java @@ -62,6 +62,10 @@ public MySQLUnaryPrefixOperation(MySQLExpression expr, MySQLUnaryPrefixOperator super(expr, op); } + public MySQLUnaryPrefixOperator getOp() { + return op; + } + @Override public MySQLConstant getExpectedValue() { MySQLConstant subExprVal = expr.getExpectedValue(); diff --git a/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java b/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java new file mode 100644 index 000000000..33f3ab587 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java @@ -0,0 +1,70 @@ +package sqlancer.mysql.gen; + +import java.util.List; + +import sqlancer.common.oracle.EETNodeFactory; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; + +/** + * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformation EET transformation} + * rules. + */ +public class MySQLEETNodeFactory implements EETNodeFactory { + + private final MySQLExpressionGenerator gen; + + public MySQLEETNodeFactory(MySQLExpressionGenerator gen) { + this.gen = gen; + } + + @Override + public MySQLExpression and(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); + } + + @Override + public MySQLExpression or(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); + } + + @Override + public MySQLExpression not(MySQLExpression expr) { + return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); + } + + @Override + public MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); + } + + @Override + public MySQLExpression isNotNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); + } + + @Override + public MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { + return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); + } + + @Override + public MySQLExpression generateBooleanExpression() { + return gen.generateBooleanExpression(); + } + + @Override + public boolean isCaseWhenApplicable(MySQLExpression expr) { + // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET + // paper); aggregates are excluded to avoid placing them in invalid contexts. + return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); + } +} diff --git a/src/sqlancer/mysql/gen/MySQLEETTransformer.java b/src/sqlancer/mysql/gen/MySQLEETTransformer.java new file mode 100644 index 000000000..000e29bb6 --- /dev/null +++ b/src/sqlancer/mysql/gen/MySQLEETTransformer.java @@ -0,0 +1,124 @@ +package sqlancer.mysql.gen; + +import java.util.List; +import java.util.stream.Collectors; + +import sqlancer.Randomly; +import sqlancer.common.oracle.EETTransformation; +import sqlancer.mysql.ast.MySQLBetweenOperation; +import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryOperation; +import sqlancer.mysql.ast.MySQLCaseOperator; +import sqlancer.mysql.ast.MySQLCastOperation; +import sqlancer.mysql.ast.MySQLComputableFunction; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; +import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; + +/** + * Recursively applies the {@link EETTransformation EET} transformation rules throughout a MySQL expression's AST. At + * each node the transformer first recurses into (and rebuilds the node from) its transformed children, then, with some + * probability, wraps the resulting sub-expression with a randomly chosen transformation rule. + * + *

+ * A boolean/scalar context flag is threaded through the recursion so that the determined-boolean rules (which reduce an + * expression to a boolean value) are only ever applied where the expression is used purely for its truth value. + */ +public class MySQLEETTransformer { + + private static final boolean BOOLEAN = true; + private static final boolean SCALAR = false; + + private final EETTransformation transformation; + + public MySQLEETTransformer(MySQLExpressionGenerator gen) { + this.transformation = new EETTransformation<>(new MySQLEETNodeFactory(gen)); + } + + /** + * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the + * root, guaranteeing that the returned expression differs from the input. + */ + public MySQLExpression transform(MySQLExpression expr, boolean booleanContext) { + return transformNode(expr, booleanContext, true); + } + + private MySQLExpression transformNode(MySQLExpression expr, boolean booleanContext, boolean forceApply) { + MySQLExpression descended = descend(expr, booleanContext); + if (forceApply || Randomly.getBoolean()) { + return transformation.applyRandomRule(descended, booleanContext); + } + return descended; + } + + /** + * Rebuilds {@code expr} with its children transformed. Leaf nodes (columns, constants, ...) and node types that are + * not rebuilt here are returned unchanged; any applicable transformation is still applied to them by the calling + * {@link #transformNode}. + */ + private MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { + if (expr instanceof MySQLBinaryLogicalOperation) { + // AND/OR/XOR: both operands are evaluated in a boolean context. + MySQLBinaryLogicalOperation op = (MySQLBinaryLogicalOperation) expr; + return new MySQLBinaryLogicalOperation(transformNode(op.getLeft(), BOOLEAN, false), + transformNode(op.getRight(), BOOLEAN, false), op.getOp()); + } else if (expr instanceof MySQLBinaryComparisonOperation) { + MySQLBinaryComparisonOperation op = (MySQLBinaryComparisonOperation) expr; + return new MySQLBinaryComparisonOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLBinaryOperation) { + MySQLBinaryOperation op = (MySQLBinaryOperation) expr; + return new MySQLBinaryOperation(transformNode(op.getLeft(), SCALAR, false), + transformNode(op.getRight(), SCALAR, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + MySQLUnaryPrefixOperation op = (MySQLUnaryPrefixOperation) expr; + boolean childContext = op.getOp() == MySQLUnaryPrefixOperator.NOT ? BOOLEAN : SCALAR; + return new MySQLUnaryPrefixOperation(transformNode(op.getExpression(), childContext, false), op.getOp()); + } else if (expr instanceof MySQLUnaryPostfixOperation) { + // The operand is transformed value-preservingly (scalar), which is safe for IS NULL/IS TRUE/IS FALSE. + MySQLUnaryPostfixOperation op = (MySQLUnaryPostfixOperation) expr; + return new MySQLUnaryPostfixOperation(transformNode(op.getExpression(), SCALAR, false), op.getOperator(), + op.isNegated()); + } else if (expr instanceof MySQLCastOperation) { + MySQLCastOperation op = (MySQLCastOperation) expr; + return new MySQLCastOperation(transformNode(op.getExpr(), SCALAR, false), op.getType()); + } else if (expr instanceof MySQLBetweenOperation) { + MySQLBetweenOperation op = (MySQLBetweenOperation) expr; + return new MySQLBetweenOperation(transformNode(op.getExpr(), SCALAR, false), + transformNode(op.getLeft(), SCALAR, false), transformNode(op.getRight(), SCALAR, false)); + } else if (expr instanceof MySQLInOperation) { + MySQLInOperation op = (MySQLInOperation) expr; + List listElements = op.getListElements().stream() + .map(e -> transformNode(e, SCALAR, false)).collect(Collectors.toList()); + return new MySQLInOperation(transformNode(op.getExpr(), SCALAR, false), listElements, op.isTrue()); + } else if (expr instanceof MySQLComputableFunction) { + MySQLComputableFunction op = (MySQLComputableFunction) expr; + MySQLExpression[] args = op.getArguments(); + MySQLExpression[] newArgs = new MySQLExpression[args.length]; + for (int i = 0; i < args.length; i++) { + newArgs[i] = transformNode(args[i], SCALAR, false); + } + return new MySQLComputableFunction(op.getFunction(), newArgs); + } else if (expr instanceof MySQLCaseOperator) { + return descendCase((MySQLCaseOperator) expr); + } + return expr; + } + + private MySQLExpression descendCase(MySQLCaseOperator caseOp) { + MySQLExpression switchCondition = caseOp.getSwitchCondition(); + // Without a switch operand the WHEN conditions are boolean; with one they are compared against the operand. + boolean conditionContext = switchCondition == null ? BOOLEAN : SCALAR; + MySQLExpression newSwitch = switchCondition == null ? null : transformNode(switchCondition, SCALAR, false); + List conditions = caseOp.getConditions().stream() + .map(e -> transformNode(e, conditionContext, false)).collect(Collectors.toList()); + List expressions = caseOp.getExpressions().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); + MySQLExpression elseExpr = caseOp.getElseExpr() == null ? null + : transformNode(caseOp.getElseExpr(), SCALAR, false); + return new MySQLCaseOperator(newSwitch, conditions, expressions, elseExpr); + } +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index d8ce5dd37..29b8d5ac5 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; import sqlancer.common.schema.AbstractTables; @@ -48,11 +49,13 @@ public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, - CERTGenerator { + CERTGenerator, + EETGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; private List tables; + private MySQLEETTransformer eetTransformer; public MySQLExpressionGenerator(MySQLGlobalState state) { this.state = state; @@ -236,6 +239,14 @@ public MySQLSelect generateSelect() { return new MySQLSelect(); } + @Override + public MySQLExpression transformExpression(MySQLExpression expr, boolean booleanContext) { + if (eetTransformer == null) { + eetTransformer = new MySQLEETTransformer(this); + } + return eetTransformer.transform(expr, booleanContext); + } + @Override public List getRandomJoinClauses() { return List.of(); diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 054a66cb6..c17ccf0d5 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -362,7 +362,7 @@ private void appendType(MySQLDataType randomType) { } if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS - || o == MySQLOracleFactory.DQP)) { + || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET)) { sb.append(" ZEROFILL"); } } From 7c9f80f81440213b7a00997e8c0605533fc06b90 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 10 Jul 2026 15:31:17 +0800 Subject: [PATCH 825/856] Clarify that the E in the EET generics must extend the Expression class --- src/sqlancer/common/oracle/EETNodeFactory.java | 4 +++- src/sqlancer/common/oracle/EETTransformation.java | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java index 2cc933ced..c446a6054 100644 --- a/src/sqlancer/common/oracle/EETNodeFactory.java +++ b/src/sqlancer/common/oracle/EETNodeFactory.java @@ -1,5 +1,7 @@ package sqlancer.common.oracle; +import sqlancer.common.ast.newast.Expression; + /** * Factory for constructing the AST nodes needed by the {@link EETTransformation equivalent expression transformation} * rules. Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface @@ -8,7 +10,7 @@ * @param * the DBMS-specific expression type */ -public interface EETNodeFactory { +public interface EETNodeFactory> { /** Builds {@code left AND right}. */ E and(E left, E right); diff --git a/src/sqlancer/common/oracle/EETTransformation.java b/src/sqlancer/common/oracle/EETTransformation.java index ab9b7fefe..ecf2eea74 100644 --- a/src/sqlancer/common/oracle/EETTransformation.java +++ b/src/sqlancer/common/oracle/EETTransformation.java @@ -1,5 +1,6 @@ package sqlancer.common.oracle; +import sqlancer.common.ast.newast.Expression; import sqlancer.Randomly; /** @@ -18,7 +19,7 @@ * @param * the DBMS-specific expression type */ -public class EETTransformation { +public class EETTransformation> { private final EETNodeFactory factory; @@ -39,7 +40,7 @@ private E falseExpr() { } /** - * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen, applicable + * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen applicable * transformation rule. * * @param expr From 95dcc8ef94c5793595ad1a2e5481fbd7411835c2 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 11 Jul 2026 20:56:54 +0800 Subject: [PATCH 826/856] Refactor EET oracle structure for consistency and extensibility Merge `EETTransformation` into a new abstract base class `EETTransformer` (template method pattern: rules + tree-walking orchestration in the base, `descend` abstract for DBMS-specific AST reconstruction). Move `MySQLEETNodeFactory` and `MySQLEETTransformer` from `mysql/gen` to `mysql/oracle` to mirror the placement of their common counterparts. `MySQLEETTransformer` now extends `EETTransformer`. --- .../common/oracle/EETNodeFactory.java | 6 +- .../common/oracle/EETTransformation.java | 84 ------------- .../common/oracle/EETTransformer.java | 112 ++++++++++++++++++ .../mysql/gen/MySQLExpressionGenerator.java | 1 + .../{gen => oracle}/MySQLEETNodeFactory.java | 6 +- .../{gen => oracle}/MySQLEETTransformer.java | 45 ++----- 6 files changed, 128 insertions(+), 126 deletions(-) delete mode 100644 src/sqlancer/common/oracle/EETTransformation.java create mode 100644 src/sqlancer/common/oracle/EETTransformer.java rename src/sqlancer/mysql/{gen => oracle}/MySQLEETNodeFactory.java (95%) rename src/sqlancer/mysql/{gen => oracle}/MySQLEETTransformer.java (72%) diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java index c446a6054..9f97ea937 100644 --- a/src/sqlancer/common/oracle/EETNodeFactory.java +++ b/src/sqlancer/common/oracle/EETNodeFactory.java @@ -3,9 +3,9 @@ import sqlancer.common.ast.newast.Expression; /** - * Factory for constructing the AST nodes needed by the {@link EETTransformation equivalent expression transformation} - * rules. Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface - * lets the (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. + * Factory for constructing the AST nodes needed by the {@link EETTransformer EET transformer's} transformation rules. + * Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface lets the + * (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. * * @param * the DBMS-specific expression type diff --git a/src/sqlancer/common/oracle/EETTransformation.java b/src/sqlancer/common/oracle/EETTransformation.java deleted file mode 100644 index ecf2eea74..000000000 --- a/src/sqlancer/common/oracle/EETTransformation.java +++ /dev/null @@ -1,84 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.common.ast.newast.Expression; -import sqlancer.Randomly; - -/** - * Implements the semantic-preserving expression transformation rules of EET (Equivalent Expression Transformation, Jiang - * & Su, OSDI'24), Table 2. Given an expression, {@link #applyRandomRule} returns a semantically equivalent - * expression built from the primitives provided by an {@link EETNodeFactory}. The rules are DBMS-independent; only the - * node construction (via the factory) is DBMS-specific. - * - *

- * The rules rely on two always-determined boolean expressions built from an arbitrary boolean {@code p}: - *

    - *
  • {@code true_expr(p) = p OR (NOT p) OR (p IS NULL)}, which always evaluates to TRUE, and
    • - *
  • {@code false_expr(p) = p AND (NOT p) AND (p IS NOT NULL)}, which always evaluates to FALSE.
    • - *
- * - * @param - * the DBMS-specific expression type - */ -public class EETTransformation> { - - private final EETNodeFactory factory; - - public EETTransformation(EETNodeFactory factory) { - this.factory = factory; - } - - // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE - private E trueExpr() { - E p = factory.generateBooleanExpression(); - return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); - } - - // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE - private E falseExpr() { - E p = factory.generateBooleanExpression(); - return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); - } - - /** - * Transforms {@code expr} into a semantically equivalent expression by applying a randomly chosen applicable - * transformation rule. - * - * @param expr - * the expression to transform - * @param booleanContext - * whether {@code expr} is evaluated purely for its truth value (e.g. a WHERE predicate or an operand of a - * logical operator). Only in a boolean context may the determined-boolean rules (No. 1 and 2), which - * reduce the expression to a boolean value, be applied; in a scalar context they would change the - * expression's value/type and are therefore excluded. - * - * @return a semantically equivalent expression - */ - public E applyRandomRule(E expr, boolean booleanContext) { - int rule; - if (booleanContext) { - // Rules No. 1-6 are all value-preserving in a boolean context. - rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); - } else { - if (!factory.isCaseWhenApplicable(expr)) { - return expr; // rule No. 7: transform the expression to itself - } - // In a scalar context only the CASE WHEN rules preserve the exact value and type. - rule = Randomly.fromOptions(3, 4, 5, 6); - } - switch (rule) { - case 1: // bool_expr => false_expr OR bool_expr - return factory.or(falseExpr(), expr); - case 2: // bool_expr => true_expr AND bool_expr - return factory.and(trueExpr(), expr); - case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END - return factory.caseWhen(falseExpr(), expr, expr); - case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END - return factory.caseWhen(trueExpr(), expr, expr); - case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END - case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END - return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); - default: - throw new AssertionError(rule); - } - } -} diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java new file mode 100644 index 000000000..d66935e18 --- /dev/null +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -0,0 +1,112 @@ +package sqlancer.common.oracle; + +import sqlancer.Randomly; +import sqlancer.common.ast.newast.Expression; + +/** + * Abstract base class for EET (Equivalent Expression Transformation) tree-walkers, based on "Detecting Logic Bugs in + * Database Engines via Equivalent Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework + * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST + * nodes from their transformed children; everything else (the rule logic, context threading, and tree-walking + * orchestration) is provided here. + * + * @param + * the DBMS-specific expression type + */ +public abstract class EETTransformer> { + + private final EETNodeFactory factory; + + protected EETTransformer(EETNodeFactory factory) { + this.factory = factory; + } + + // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE + private E trueExpr() { + E p = factory.generateBooleanExpression(); + return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); + } + + // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE + private E falseExpr() { + E p = factory.generateBooleanExpression(); + return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); + } + + /** + * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent + * expression. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value; only in a boolean context may the + * determined-boolean rules (No. 1 and 2), which reduce the expression to a boolean value, be applied + * + * @return a semantically equivalent expression + */ + protected E applyRandomRule(E expr, boolean booleanContext) { + int rule; + if (booleanContext) { + // Rules No. 1-6 are all value-preserving in a boolean context. + rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); + } else { + if (!factory.isCaseWhenApplicable(expr)) { + return expr; // rule No. 7: transform the expression to itself + } + // In a scalar context only the CASE WHEN rules preserve the exact value and type. + rule = Randomly.fromOptions(3, 4, 5, 6); + } + switch (rule) { + case 1: // bool_expr => false_expr OR bool_expr + return factory.or(falseExpr(), expr); + case 2: // bool_expr => true_expr AND bool_expr + return factory.and(trueExpr(), expr); + case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END + return factory.caseWhen(falseExpr(), expr, expr); + case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END + return factory.caseWhen(trueExpr(), expr, expr); + case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END + case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END + return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); + default: + throw new AssertionError(rule); + } + } + + /** + * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the + * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. + */ + public E transform(E expr, boolean booleanContext) { + return transformNode(expr, booleanContext, true); + } + + /** + * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node. + */ + protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { + E descended = descend(expr, booleanContext); + if (forceApply || Randomly.getBoolean()) { + return applyRandomRule(descended, booleanContext); + } + return descended; + } + + /** + * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each child. + * Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable transformation + * will still be applied to them by the calling {@link #transformNode}. + * + * @param expr + * the expression to descend into + * @param booleanContext + * the context in which {@code expr} itself is evaluated (used to determine child contexts) + * + * @return a rebuilt copy of {@code expr} with transformed children, or {@code expr} itself if it is a leaf + */ + protected abstract E descend(E expr, boolean booleanContext); +} diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 29b8d5ac5..904013a37 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -46,6 +46,7 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, diff --git a/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java b/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java similarity index 95% rename from src/sqlancer/mysql/gen/MySQLEETNodeFactory.java rename to src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java index 33f3ab587..c73b576c5 100644 --- a/src/sqlancer/mysql/gen/MySQLEETNodeFactory.java +++ b/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java @@ -1,4 +1,4 @@ -package sqlancer.mysql.gen; +package sqlancer.mysql.oracle; import java.util.List; @@ -13,10 +13,10 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.gen.MySQLExpressionGenerator; /** - * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformation EET transformation} - * rules. + * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformer EET transformer}. */ public class MySQLEETNodeFactory implements EETNodeFactory { diff --git a/src/sqlancer/mysql/gen/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java similarity index 72% rename from src/sqlancer/mysql/gen/MySQLEETTransformer.java rename to src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 000e29bb6..71af94ed8 100644 --- a/src/sqlancer/mysql/gen/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -1,10 +1,9 @@ -package sqlancer.mysql.gen; +package sqlancer.mysql.oracle; import java.util.List; import java.util.stream.Collectors; -import sqlancer.Randomly; -import sqlancer.common.oracle.EETTransformation; +import sqlancer.common.oracle.EETTransformer; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -17,49 +16,23 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.mysql.gen.MySQLExpressionGenerator; /** - * Recursively applies the {@link EETTransformation EET} transformation rules throughout a MySQL expression's AST. At - * each node the transformer first recurses into (and rebuilds the node from) its transformed children, then, with some - * probability, wraps the resulting sub-expression with a randomly chosen transformation rule. - * - *

- * A boolean/scalar context flag is threaded through the recursion so that the determined-boolean rules (which reduce an - * expression to a boolean value) are only ever applied where the expression is used purely for its truth value. + * MySQL implementation of the {@link EETTransformer EET} tree-walker. Implements {@link #descend} to rebuild MySQL AST + * nodes from their transformed children, threading the correct boolean/scalar context into each child. */ -public class MySQLEETTransformer { +public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; - private final EETTransformation transformation; - public MySQLEETTransformer(MySQLExpressionGenerator gen) { - this.transformation = new EETTransformation<>(new MySQLEETNodeFactory(gen)); - } - - /** - * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the - * root, guaranteeing that the returned expression differs from the input. - */ - public MySQLExpression transform(MySQLExpression expr, boolean booleanContext) { - return transformNode(expr, booleanContext, true); - } - - private MySQLExpression transformNode(MySQLExpression expr, boolean booleanContext, boolean forceApply) { - MySQLExpression descended = descend(expr, booleanContext); - if (forceApply || Randomly.getBoolean()) { - return transformation.applyRandomRule(descended, booleanContext); - } - return descended; + super(new MySQLEETNodeFactory(gen)); } - /** - * Rebuilds {@code expr} with its children transformed. Leaf nodes (columns, constants, ...) and node types that are - * not rebuilt here are returned unchanged; any applicable transformation is still applied to them by the calling - * {@link #transformNode}. - */ - private MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { + @Override + protected MySQLExpression descend(MySQLExpression expr, boolean booleanContext) { if (expr instanceof MySQLBinaryLogicalOperation) { // AND/OR/XOR: both operands are evaluated in a boolean context. MySQLBinaryLogicalOperation op = (MySQLBinaryLogicalOperation) expr; From 657ec00a07e21a523924b7433283f780bbb07aee Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 09:49:12 +0800 Subject: [PATCH 827/856] Move EET transformer ownership to EETOracle, removing EET-specific state from DBMS-specific expression generators --- src/sqlancer/common/gen/EETGenerator.java | 20 +++++++------------ src/sqlancer/common/oracle/EETOracle.java | 6 ++++-- .../mysql/gen/MySQLExpressionGenerator.java | 9 +++------ 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index f468ff6d2..60b91e78a 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -5,14 +5,15 @@ import sqlancer.common.ast.newast.Expression; import sqlancer.common.ast.newast.Join; import sqlancer.common.ast.newast.Select; +import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; import sqlancer.common.schema.AbstractTables; /** * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like - * the other oracle generators), an EET generator can transform an expression into a semantically equivalent one - * according to the EET transformation rules. + * the other oracle generators), an EET generator creates a DBMS-specific {@link EETTransformer} that the oracle uses to + * rewrite expressions into semantically equivalent ones. */ public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { @@ -29,16 +30,9 @@ public interface EETGenerator, J extends Join createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 0f192f9ab..0ae229414 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -32,6 +32,7 @@ public class EETOracle, J extends Join, E private final G state; private EETGenerator gen; + private final EETTransformer transformer; private final ExpectedErrors errors; private Reproducer reproducer; @@ -69,6 +70,7 @@ public EETOracle(G state, EETGenerator gen, ExpectedErrors expect } this.state = state; this.gen = gen; + this.transformer = gen.createTransformer(); this.errors = expectedErrors; } @@ -94,10 +96,10 @@ public void check() throws SQLException { // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, // while the WHERE clause is evaluated in a boolean context. - List transformedFetchColumns = fetchColumns.stream().map(c -> gen.transformExpression(c, false)) + List transformedFetchColumns = fetchColumns.stream().map(c -> transformer.transform(c, false)) .collect(Collectors.toList()); select.setFetchColumns(transformedFetchColumns); - select.setWhereClause(gen.transformExpression(whereClause, true)); + select.setWhereClause(transformer.transform(whereClause, true)); String transformedQueryString = select.asString(); List transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 904013a37..f2d577aee 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -46,6 +46,7 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; +import sqlancer.common.oracle.EETTransformer; import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator @@ -56,7 +57,6 @@ public class MySQLExpressionGenerator extends UntypedExpressionGenerator tables; - private MySQLEETTransformer eetTransformer; public MySQLExpressionGenerator(MySQLGlobalState state) { this.state = state; @@ -241,11 +241,8 @@ public MySQLSelect generateSelect() { } @Override - public MySQLExpression transformExpression(MySQLExpression expr, boolean booleanContext) { - if (eetTransformer == null) { - eetTransformer = new MySQLEETTransformer(this); - } - return eetTransformer.transform(expr, booleanContext); + public EETTransformer createTransformer() { + return new MySQLEETTransformer(this); } @Override From 65957adf9b8fb4b2702c20475c9c5d2e5c972560 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 10:03:19 +0800 Subject: [PATCH 828/856] Rearrange MySQLExpressionGenerator code to group oracle-specific sections together --- .../mysql/gen/MySQLExpressionGenerator.java | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index f2d577aee..5d1ec06f4 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -222,6 +222,22 @@ public List generateOrderBys() { return newOrderBys; } + public MySQLAggregate generateAggregate() { + MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); + + if (func.isVariadic()) { + int nrExprs = Randomly.smallNumber() + 1; + List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) + .collect(Collectors.toList()); + + return new MySQLAggregate(exprs, func); + } else { + return new MySQLAggregate(List.of(generateExpression()), func); + } + } + + // --- Shared oracle infrastructure (TLPWhere / CERT / EET) --- + @Override public MySQLExpressionGenerator setTablesAndColumns(AbstractTables tables) { this.columns = tables.getColumns(); @@ -240,11 +256,6 @@ public MySQLSelect generateSelect() { return new MySQLSelect(); } - @Override - public EETTransformer createTransformer() { - return new MySQLEETTransformer(this); - } - @Override public List getRandomJoinClauses() { return List.of(); @@ -260,6 +271,8 @@ public List generateFetchColumns(boolean shouldCreateDummy) { return columns.stream().map(c -> new MySQLColumnReference(c, null)).collect(Collectors.toList()); } + // --- CERT oracle --- + @Override public String generateExplainQuery(MySQLSelect select) { return "EXPLAIN FORMAT=TRADITIONAL " + select.asString(); // as of MySQL 9.5.0, default EXPLAIN format changed @@ -267,20 +280,6 @@ public String generateExplainQuery(MySQLSelect select) { // now be specified } - public MySQLAggregate generateAggregate() { - MySQLAggregateFunction func = Randomly.fromOptions(MySQLAggregateFunction.values()); - - if (func.isVariadic()) { - int nrExprs = Randomly.smallNumber() + 1; - List exprs = IntStream.range(0, nrExprs).mapToObj(index -> generateExpression()) - .collect(Collectors.toList()); - - return new MySQLAggregate(exprs, func); - } else { - return new MySQLAggregate(List.of(generateExpression()), func); - } - } - @Override public boolean mutate(MySQLSelect select) { List> mutators = new ArrayList<>(); @@ -364,4 +363,11 @@ boolean mutateOr(MySQLSelect select) { return true; } } + + // --- EET oracle --- + + @Override + public EETTransformer createTransformer() { + return new MySQLEETTransformer(this); + } } From 03c487476cae5d040de76abff895c1fe5c79ceae Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 10:24:34 +0800 Subject: [PATCH 829/856] Remove need for EETNodeFactory by merging it into EETTransformer --- .../common/oracle/EETNodeFactory.java | 41 ----------- .../common/oracle/EETTransformer.java | 57 ++++++++++----- .../mysql/oracle/MySQLEETNodeFactory.java | 70 ------------------- .../mysql/oracle/MySQLEETTransformer.java | 50 ++++++++++++- 4 files changed, 88 insertions(+), 130 deletions(-) delete mode 100644 src/sqlancer/common/oracle/EETNodeFactory.java delete mode 100644 src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java diff --git a/src/sqlancer/common/oracle/EETNodeFactory.java b/src/sqlancer/common/oracle/EETNodeFactory.java deleted file mode 100644 index 9f97ea937..000000000 --- a/src/sqlancer/common/oracle/EETNodeFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -package sqlancer.common.oracle; - -import sqlancer.common.ast.newast.Expression; - -/** - * Factory for constructing the AST nodes needed by the {@link EETTransformer EET transformer's} transformation rules. - * Because every DBMS has its own expression AST, the actual node construction is DBMS-specific; this interface lets the - * (DBMS-independent) transformation rules be expressed once in terms of a small set of primitive operations. - * - * @param - * the DBMS-specific expression type - */ -public interface EETNodeFactory> { - - /** Builds {@code left AND right}. */ - E and(E left, E right); - - /** Builds {@code left OR right}. */ - E or(E left, E right); - - /** Builds {@code NOT expr}. */ - E not(E expr); - - /** Builds {@code expr IS NULL}. */ - E isNull(E expr); - - /** Builds {@code expr IS NOT NULL}. */ - E isNotNull(E expr); - - /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ - E caseWhen(E condition, E thenExpr, E elseExpr); - - /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ - E generateBooleanExpression(); - - /** - * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not - * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). - */ - boolean isCaseWhenApplicable(E expr); -} diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index d66935e18..010cd0a56 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -10,30 +10,24 @@ *

* This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST - * nodes from their transformed children; everything else (the rule logic, context threading, and tree-walking - * orchestration) is provided here. + * nodes from their transformed children, and the abstract factory methods to construct new nodes; everything else (the + * rule logic, context threading, and tree-walking orchestration) is provided here. * * @param * the DBMS-specific expression type */ public abstract class EETTransformer> { - private final EETNodeFactory factory; - - protected EETTransformer(EETNodeFactory factory) { - this.factory = factory; - } - // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE private E trueExpr() { - E p = factory.generateBooleanExpression(); - return factory.or(factory.or(p, factory.not(p)), factory.isNull(p)); + E p = generateBooleanExpression(); + return or(or(p, not(p)), isNull(p)); } // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE private E falseExpr() { - E p = factory.generateBooleanExpression(); - return factory.and(factory.and(p, factory.not(p)), factory.isNotNull(p)); + E p = generateBooleanExpression(); + return and(and(p, not(p)), isNotNull(p)); } /** @@ -54,7 +48,7 @@ protected E applyRandomRule(E expr, boolean booleanContext) { // Rules No. 1-6 are all value-preserving in a boolean context. rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); } else { - if (!factory.isCaseWhenApplicable(expr)) { + if (!isCaseWhenApplicable(expr)) { return expr; // rule No. 7: transform the expression to itself } // In a scalar context only the CASE WHEN rules preserve the exact value and type. @@ -62,16 +56,16 @@ protected E applyRandomRule(E expr, boolean booleanContext) { } switch (rule) { case 1: // bool_expr => false_expr OR bool_expr - return factory.or(falseExpr(), expr); + return or(falseExpr(), expr); case 2: // bool_expr => true_expr AND bool_expr - return factory.and(trueExpr(), expr); + return and(trueExpr(), expr); case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END - return factory.caseWhen(falseExpr(), expr, expr); + return caseWhen(falseExpr(), expr, expr); case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END - return factory.caseWhen(trueExpr(), expr, expr); + return caseWhen(trueExpr(), expr, expr); case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END - return factory.caseWhen(factory.generateBooleanExpression(), expr, expr); + return caseWhen(generateBooleanExpression(), expr, expr); default: throw new AssertionError(rule); } @@ -109,4 +103,31 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * @return a rebuilt copy of {@code expr} with transformed children, or {@code expr} itself if it is a leaf */ protected abstract E descend(E expr, boolean booleanContext); + + /** Builds {@code left AND right}. */ + protected abstract E and(E left, E right); + + /** Builds {@code left OR right}. */ + protected abstract E or(E left, E right); + + /** Builds {@code NOT expr}. */ + protected abstract E not(E expr); + + /** Builds {@code expr IS NULL}. */ + protected abstract E isNull(E expr); + + /** Builds {@code expr IS NOT NULL}. */ + protected abstract E isNotNull(E expr); + + /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ + protected abstract E caseWhen(E condition, E thenExpr, E elseExpr); + + /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ + protected abstract E generateBooleanExpression(); + + /** + * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not + * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + */ + protected abstract boolean isCaseWhenApplicable(E expr); } diff --git a/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java b/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java deleted file mode 100644 index c73b576c5..000000000 --- a/src/sqlancer/mysql/oracle/MySQLEETNodeFactory.java +++ /dev/null @@ -1,70 +0,0 @@ -package sqlancer.mysql.oracle; - -import java.util.List; - -import sqlancer.common.oracle.EETNodeFactory; -import sqlancer.mysql.ast.MySQLAggregate; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; -import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; -import sqlancer.mysql.ast.MySQLCaseOperator; -import sqlancer.mysql.ast.MySQLExpression; -import sqlancer.mysql.ast.MySQLTableReference; -import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; -import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; -import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; -import sqlancer.mysql.gen.MySQLExpressionGenerator; - -/** - * Constructs the MySQL AST nodes needed by the {@link sqlancer.common.oracle.EETTransformer EET transformer}. - */ -public class MySQLEETNodeFactory implements EETNodeFactory { - - private final MySQLExpressionGenerator gen; - - public MySQLEETNodeFactory(MySQLExpressionGenerator gen) { - this.gen = gen; - } - - @Override - public MySQLExpression and(MySQLExpression left, MySQLExpression right) { - return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); - } - - @Override - public MySQLExpression or(MySQLExpression left, MySQLExpression right) { - return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); - } - - @Override - public MySQLExpression not(MySQLExpression expr) { - return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); - } - - @Override - public MySQLExpression isNull(MySQLExpression expr) { - return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); - } - - @Override - public MySQLExpression isNotNull(MySQLExpression expr) { - return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); - } - - @Override - public MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { - return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); - } - - @Override - public MySQLExpression generateBooleanExpression() { - return gen.generateBooleanExpression(); - } - - @Override - public boolean isCaseWhenApplicable(MySQLExpression expr) { - // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET - // paper); aggregates are excluded to avoid placing them in invalid contexts. - return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); - } -} diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 71af94ed8..9b0cac226 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -4,16 +4,20 @@ import java.util.stream.Collectors; import sqlancer.common.oracle.EETTransformer; +import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; import sqlancer.mysql.ast.MySQLBinaryOperation; import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; import sqlancer.mysql.ast.MySQLComputableFunction; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLUnaryPostfixOperation.UnaryPostfixOperator; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; import sqlancer.mysql.gen.MySQLExpressionGenerator; @@ -27,8 +31,10 @@ public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; + private final MySQLExpressionGenerator gen; + public MySQLEETTransformer(MySQLExpressionGenerator gen) { - super(new MySQLEETNodeFactory(gen)); + this.gen = gen; } @Override @@ -94,4 +100,46 @@ private MySQLExpression descendCase(MySQLCaseOperator caseOp) { : transformNode(caseOp.getElseExpr(), SCALAR, false); return new MySQLCaseOperator(newSwitch, conditions, expressions, elseExpr); } + + @Override + protected MySQLExpression and(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.AND); + } + + @Override + protected MySQLExpression or(MySQLExpression left, MySQLExpression right) { + return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); + } + + @Override + protected MySQLExpression not(MySQLExpression expr) { + return new MySQLUnaryPrefixOperation(expr, MySQLUnaryPrefixOperator.NOT); + } + + @Override + protected MySQLExpression isNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, false); + } + + @Override + protected MySQLExpression isNotNull(MySQLExpression expr) { + return new MySQLUnaryPostfixOperation(expr, UnaryPostfixOperator.IS_NULL, true); + } + + @Override + protected MySQLExpression caseWhen(MySQLExpression condition, MySQLExpression thenExpr, MySQLExpression elseExpr) { + return new MySQLCaseOperator(null, List.of(condition), List.of(thenExpr), elseExpr); + } + + @Override + protected MySQLExpression generateBooleanExpression() { + return gen.generateBooleanExpression(); + } + + @Override + protected boolean isCaseWhenApplicable(MySQLExpression expr) { + // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET + // paper); aggregates are excluded to avoid placing them in invalid contexts. + return !(expr instanceof MySQLTableReference) && !(expr instanceof MySQLAggregate); + } } From 9285e9168dd3a3d06cc080b6cc75d46aea216529 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 12 Jul 2026 18:17:44 +0800 Subject: [PATCH 830/856] Correct EET transformation rule comments --- src/sqlancer/common/oracle/EETTransformer.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index 010cd0a56..12999402c 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -55,17 +55,18 @@ protected E applyRandomRule(E expr, boolean booleanContext) { rule = Randomly.fromOptions(3, 4, 5, 6); } switch (rule) { - case 1: // bool_expr => false_expr OR bool_expr + case 1: // expr => false_expr OR expr return or(falseExpr(), expr); - case 2: // bool_expr => true_expr AND bool_expr + case 2: // expr => true_expr AND expr return and(trueExpr(), expr); - case 3: // expr => CASE WHEN false_expr THEN copy(expr) ELSE expr END + case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END return caseWhen(falseExpr(), expr, expr); - case 4: // expr => CASE WHEN true_expr THEN expr ELSE copy(expr) END + case 4: // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END return caseWhen(trueExpr(), expr, expr); - case 5: // expr => CASE WHEN rand_bool THEN copy(expr) ELSE expr END - case 6: // expr => CASE WHEN rand_bool THEN expr ELSE copy(expr) END - return caseWhen(generateBooleanExpression(), expr, expr); + case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END + case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END + return caseWhen(generateBooleanExpression(), expr, expr); + // deep copy of expr is not needed, as the AST nodes are immutable anyway default: throw new AssertionError(rule); } From 52ce86661581681b1eb754ebcdc89103c833fa76 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Mon, 13 Jul 2026 12:40:14 +0800 Subject: [PATCH 831/856] Implement rules 3 and 4 for EET --- src/sqlancer/common/gen/EETGenerator.java | 2 +- src/sqlancer/common/oracle/EETOracle.java | 2 +- .../common/oracle/EETTransformer.java | 51 ++++++- .../mysql/ast/MySQLCastOperation.java | 7 +- .../mysql/gen/MySQLExpressionGenerator.java | 2 +- .../mysql/oracle/MySQLEETTransformer.java | 128 +++++++++++++++++- 6 files changed, 179 insertions(+), 13 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index 60b91e78a..85aaa7462 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -34,5 +34,5 @@ public interface EETGenerator, J extends Join createTransformer(); + EETTransformer createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 0ae229414..148532e9b 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -32,7 +32,7 @@ public class EETOracle, J extends Join, E private final G state; private EETGenerator gen; - private final EETTransformer transformer; + private final EETTransformer transformer; private final ExpectedErrors errors; private Reproducer reproducer; diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index 12999402c..f27a917b9 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -10,13 +10,16 @@ *

* This class implements the seven transformation rules (Table 2 of the paper) and provides a template-method framework * for applying them throughout an expression's AST. Subclasses implement {@link #descend} to rebuild DBMS-specific AST - * nodes from their transformed children, and the abstract factory methods to construct new nodes; everything else (the - * rule logic, context threading, and tree-walking orchestration) is provided here. + * nodes from their transformed children, the abstract factory methods to construct new nodes, and the type hooks + * ({@link #inferType} and {@link #generateExpressionOfType}) that realize the paper's {@code rand_expr(type(expr))}; + * everything else (the rule logic, context threading, and tree-walking orchestration) is provided here. * * @param - * the DBMS-specific expression type + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain used by {@link #inferType} and {@link #generateExpressionOfType} */ -public abstract class EETTransformer> { +public abstract class EETTransformer, T> { // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE private E trueExpr() { @@ -30,6 +33,22 @@ private E falseExpr() { return and(and(p, not(p)), isNotNull(p)); } + /** + * Implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches that of + * {@code expr}. Although the generated expression is never evaluated (it occupies the dead branch of rules No. 3 + * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could + * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to + * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form + * of rules No. 5 and 6). + */ + private E randExprOfSameType(E expr) { + T type = inferType(expr); + if (type == null) { + return expr; + } + return generateExpressionOfType(type); + } + /** * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent * expression. @@ -60,12 +79,12 @@ protected E applyRandomRule(E expr, boolean booleanContext) { case 2: // expr => true_expr AND expr return and(trueExpr(), expr); case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END - return caseWhen(falseExpr(), expr, expr); + return caseWhen(falseExpr(), randExprOfSameType(expr), expr); case 4: // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END - return caseWhen(trueExpr(), expr, expr); + return caseWhen(trueExpr(), expr, randExprOfSameType(expr)); case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END - return caseWhen(generateBooleanExpression(), expr, expr); + return caseWhen(generateBooleanExpression(), expr, expr); // deep copy of expr is not needed, as the AST nodes are immutable anyway default: throw new AssertionError(rule); @@ -126,6 +145,24 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ protected abstract E generateBooleanExpression(); + /** + * Infers the static type of {@code expr}, or returns {@code null} if it cannot be determined. The type domain + * {@code T} is DBMS-specific and may be coarse: it only needs to be precise enough that replacing an expression + * with another of the same {@code T} leaves the DBMS's CASE WHEN result-type resolution unaffected. Returning + * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead + * branch. Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is + * uncertain. + */ + protected abstract T inferType(E expr); + + /** + * Generates a fresh random expression of static type {@code type}, reusing the variables available to the query + * generator. DBMSs with a typed expression generator can delegate to it directly; DBMSs with an untyped generator + * can instead wrap an arbitrary random expression in a CAST to {@code type} (which requires every value of + * {@code T} to be a valid CAST target). + */ + protected abstract E generateExpressionOfType(T type); + /** * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index 8ae783fa0..8a457d0f1 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -6,11 +6,14 @@ public class MySQLCastOperation implements MySQLExpression { private final CastType type; public enum CastType { - SIGNED, UNSIGNED; + SIGNED, UNSIGNED, + // CHAR and DOUBLE are used only by the EET oracle's type-pinning casts and are never evaluated, so + // MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + CHAR, DOUBLE; public static CastType getRandom() { return SIGNED; - // return Randomly.fromOptions(CastType.values()); + // return Randomly.fromOptions(CastType.SIGNED, CastType.UNSIGNED); } } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 5d1ec06f4..9b019a6d4 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -367,7 +367,7 @@ boolean mutateOr(MySQLSelect select) { // --- EET oracle --- @Override - public EETTransformer createTransformer() { + public EETTransformer createTransformer() { return new MySQLEETTransformer(this); } } diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 9b0cac226..d16a8e517 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -1,5 +1,6 @@ package sqlancer.mysql.oracle; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -12,7 +13,11 @@ import sqlancer.mysql.ast.MySQLBinaryOperation; import sqlancer.mysql.ast.MySQLCaseOperator; import sqlancer.mysql.ast.MySQLCastOperation; +import sqlancer.mysql.ast.MySQLCastOperation.CastType; +import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLComputableFunction; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; import sqlancer.mysql.ast.MySQLInOperation; import sqlancer.mysql.ast.MySQLTableReference; @@ -25,8 +30,13 @@ /** * MySQL implementation of the {@link EETTransformer EET} tree-walker. Implements {@link #descend} to rebuild MySQL AST * nodes from their transformed children, threading the correct boolean/scalar context into each child. + * + *

+ * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target types + * ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning {@code null} + * when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping it in a CAST. */ -public class MySQLEETTransformer extends EETTransformer { +public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; @@ -136,6 +146,122 @@ protected MySQLExpression generateBooleanExpression() { return gen.generateBooleanExpression(); } + @Override + protected MySQLExpression generateExpressionOfType(CastType type) { + // The MySQL expression generator is untyped, so the type of an arbitrary random expression is pinned by + // wrapping it in a CAST to the requested type. + return new MySQLCastOperation(gen.generateExpression(), type); + } + + @Override + protected CastType inferType(MySQLExpression expr) { + if (expr instanceof MySQLBinaryLogicalOperation || expr instanceof MySQLBinaryComparisonOperation + || expr instanceof MySQLUnaryPostfixOperation || expr instanceof MySQLBetweenOperation + || expr instanceof MySQLInOperation || expr instanceof MySQLExists) { + // Predicates evaluate to the boolean values 0/1, which are signed BIGINT. + return CastType.SIGNED; + } else if (expr instanceof MySQLBinaryOperation) { + // The bit operators &, | and ^ return BIGINT UNSIGNED. + return CastType.UNSIGNED; + } else if (expr instanceof MySQLCastOperation) { + return ((MySQLCastOperation) expr).getType(); + } else if (expr instanceof MySQLUnaryPrefixOperation) { + return inferUnaryPrefixType((MySQLUnaryPrefixOperation) expr); + } else if (expr instanceof MySQLConstant) { + return inferConstantType((MySQLConstant) expr); + } else if (expr instanceof MySQLColumnReference) { + return inferColumnType((MySQLColumnReference) expr); + } else if (expr instanceof MySQLComputableFunction) { + return inferFunctionType((MySQLComputableFunction) expr); + } else if (expr instanceof MySQLCaseOperator) { + return inferCaseType((MySQLCaseOperator) expr); + } + return null; + } + + private CastType inferUnaryPrefixType(MySQLUnaryPrefixOperation op) { + if (op.getOp() == MySQLUnaryPrefixOperator.NOT) { + return CastType.SIGNED; + } + CastType operandType = inferType(op.getExpression()); + if (op.getOp() == MySQLUnaryPrefixOperator.PLUS) { + return operandType; // unary + is the identity + } + // Unary -: strings are coerced to DOUBLE; negating UNSIGNED changes the type (and usually errors). + if (operandType == CastType.CHAR || operandType == CastType.DOUBLE) { + return CastType.DOUBLE; + } + return operandType == CastType.SIGNED ? CastType.SIGNED : null; + } + + private CastType inferConstantType(MySQLConstant constant) { + if (constant instanceof MySQLConstant.MySQLIntConstant) { + return constant.isSigned() ? CastType.SIGNED : CastType.UNSIGNED; + } else if (constant instanceof MySQLConstant.MySQLTextConstant) { + return CastType.CHAR; + } else if (constant instanceof MySQLConstant.MySQLDoubleConstant) { + return CastType.DOUBLE; + } + return null; // the NULL constant has no type of its own + } + + private CastType inferColumnType(MySQLColumnReference ref) { + switch (ref.getColumn().getType()) { + case INT: + return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns + case VARCHAR: + return CastType.CHAR; + case DOUBLE: + return CastType.DOUBLE; + case FLOAT: // FLOAT-to-DOUBLE widening in the CASE result changes the rendered value + case DECIMAL: // the CASE result would need the column's exact precision and scale + default: + return null; + } + } + + private CastType inferFunctionType(MySQLComputableFunction func) { + MySQLExpression[] args = func.getArguments(); + switch (func.getFunction()) { + case BIT_COUNT: + return CastType.SIGNED; + case IF: + // The result type aggregates the types of the two value arguments (the condition does not contribute). + return commonType(args[1], args[2]); + case COALESCE: + case IFNULL: + case LEAST: + case GREATEST: + return commonType(args); + default: + return null; + } + } + + private CastType inferCaseType(MySQLCaseOperator caseOp) { + List branches = new ArrayList<>(caseOp.getExpressions()); + if (caseOp.getElseExpr() != null) { + branches.add(caseOp.getElseExpr()); + } + return commonType(branches.toArray(new MySQLExpression[0])); + } + + /** + * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the + * same inferrable type (a conservative under-approximation of MySQL's aggregation rules). + */ + private CastType commonType(MySQLExpression... exprs) { + CastType common = null; + for (MySQLExpression expr : exprs) { + CastType type = inferType(expr); + if (type == null || common != null && type != common) { + return null; + } + common = type; + } + return common; + } + @Override protected boolean isCaseWhenApplicable(MySQLExpression expr) { // Table references cannot be wrapped in CASE WHEN (they would cause syntax errors, see rule No. 7 of the EET From 86e5b719a6b7fffc548e37ad15a6dd54657797c4 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 15 Jul 2026 09:50:26 +0800 Subject: [PATCH 832/856] Fix EET reproducer/reduction to port the same fixes to it that were earlier made to TLP WHERE --- src/sqlancer/common/oracle/EETOracle.java | 92 +++++++++++++++++++---- 1 file changed, 78 insertions(+), 14 deletions(-) diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 148532e9b..5af0cd1b3 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -40,28 +40,73 @@ public class EETOracle, J extends Join, E private final class EETReproducer implements Reproducer { private final String originalQueryString; + // null if the original bug was a DBMS error on the original query alone private final String transformedQueryString; - private final List resultSet; + // null if the original bug is a result set mismatch; otherwise, the message of the + // unexpected DBMS error that the original or transformed query triggered + private final String expectedErrorMessage; - EETReproducer(String originalQueryString, String transformedQueryString, List resultSet) { + EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; - this.resultSet = resultSet; + this.expectedErrorMessage = expectedErrorMessage; } @Override public boolean bugStillTriggers(G globalState) { + List originalResultSet; + List transformedResultSet; try { - List transformedResultSet = ComparatorHelper - .getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); - ComparatorHelper.assumeResultSetsAreEqual(resultSet, transformedResultSet, originalQueryString, + // Re-execute both queries against the current (reduced) database instead of comparing + // against a cached result set, which would be stale once statements have been removed. + originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, + globalState); + if (transformedQueryString == null) { + // the original bug was a DBMS error on the original query alone, which no + // longer occurs + return false; + } + transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, + globalState); + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs + return false; + } + try { + ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), globalState); - } catch (AssertionError triggeredError) { + } catch (AssertionError resultSetMismatch) { return true; - } catch (SQLException ignored) { } return false; } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage == null) { + sb.append("-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:").append(System.lineSeparator()); + } else { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage) + .append(System.lineSeparator()); + } + sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (transformedQueryString != null) { + sb.append("-- transformed: ").append(transformedQueryString).append(';') + .append(System.lineSeparator()); + } + return sb.toString(); + } } public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { @@ -91,8 +136,16 @@ public void check() throws SQLException { String originalQueryString = select.asString(); generatedQueryString = originalQueryString; - List originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - state); + List originalResultSet; + try { + originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); + } catch (AssertionError unexpectedError) { + // an unexpected DBMS error on the original query alone is itself a bug worth reducing; + // transformedQueryString is null because no transformed query is involved + reproducer = new EETReproducer(originalQueryString, null, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } // Transform the query's expressions into semantically equivalent ones. Fetch columns are scalar expressions, // while the WHERE clause is evaluated in a boolean context. @@ -102,13 +155,24 @@ public void check() throws SQLException { select.setWhereClause(transformer.transform(whereClause, true)); String transformedQueryString = select.asString(); - List transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, - errors, state); + List transformedResultSet; + try { + transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, + state); + } catch (AssertionError unexpectedError) { + // the semantics-preserving transformation made the query trigger a DBMS error that the + // original did not, which is a bug worth reducing + reproducer = new EETReproducer(originalQueryString, transformedQueryString, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + // Set the reproducer before the assertion: assumeResultSetsAreEqual throws when the bug is + // detected, so creating the reproducer afterwards would leave it null and prevent any reduction. + reproducer = new EETReproducer(originalQueryString, transformedQueryString, null); ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), state); - - reproducer = new EETReproducer(originalQueryString, transformedQueryString, originalResultSet); } @Override From 912197dba19e2f6321d9881cd815172c779dec73 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 23 Jul 2026 15:48:24 +0800 Subject: [PATCH 833/856] Add FLOAT and DECIMAL to MySQL EET target cast types --- src/sqlancer/common/gen/EETGenerator.java | 6 +++--- src/sqlancer/common/oracle/EETOracle.java | 9 ++++----- .../common/oracle/EETTransformer.java | 16 +++++++-------- .../mysql/ast/MySQLCastOperation.java | 6 +++--- .../mysql/oracle/MySQLEETTransformer.java | 20 +++++++++++++------ 5 files changed, 32 insertions(+), 25 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index 85aaa7462..ef436d975 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -30,9 +30,9 @@ public interface EETGenerator, J extends Join createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 5af0cd1b3..1422d5b6b 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -23,9 +23,9 @@ * *

* The oracle generates a random query and then transforms its expressions (the WHERE predicate and the fetch columns) - * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation preserves - * semantics, the original and the transformed query must return the same result set; any discrepancy indicates a logic - * bug in the DBMS. + * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation + * preserves semantics, the original and the transformed query must return the same result set; any discrepancy + * indicates a logic bug in the DBMS. */ public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> implements TestOracle { @@ -102,8 +102,7 @@ public String getBugInformation() { } sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); if (transformedQueryString != null) { - sb.append("-- transformed: ").append(transformedQueryString).append(';') - .append(System.lineSeparator()); + sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); } return sb.toString(); } diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index f27a917b9..932a6fb8e 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -38,8 +38,8 @@ private E falseExpr() { * {@code expr}. Although the generated expression is never evaluated (it occupies the dead branch of rules No. 3 * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to - * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form - * of rules No. 5 and 6). + * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form of + * rules No. 5 and 6). */ private E randExprOfSameType(E expr) { T type = inferType(expr); @@ -85,7 +85,7 @@ protected E applyRandomRule(E expr, boolean booleanContext) { case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END return caseWhen(generateBooleanExpression(), expr, expr); - // deep copy of expr is not needed, as the AST nodes are immutable anyway + // deep copy of expr is not needed, as the AST nodes are immutable anyway default: throw new AssertionError(rule); } @@ -111,9 +111,9 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { } /** - * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each child. - * Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable transformation - * will still be applied to them by the calling {@link #transformNode}. + * Rebuilds {@code expr} with its children transformed, threading the correct boolean/scalar context into each + * child. Leaf nodes (columns, constants, table references, ...) should be returned unchanged; any applicable + * transformation will still be applied to them by the calling {@link #transformNode}. * * @param expr * the expression to descend into @@ -149,8 +149,8 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * Infers the static type of {@code expr}, or returns {@code null} if it cannot be determined. The type domain * {@code T} is DBMS-specific and may be coarse: it only needs to be precise enough that replacing an expression * with another of the same {@code T} leaves the DBMS's CASE WHEN result-type resolution unaffected. Returning - * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead - * branch. Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is + * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead branch. + * Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is * uncertain. */ protected abstract T inferType(E expr); diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index 8a457d0f1..b71d0498c 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -7,9 +7,9 @@ public class MySQLCastOperation implements MySQLExpression { public enum CastType { SIGNED, UNSIGNED, - // CHAR and DOUBLE are used only by the EET oracle's type-pinning casts and are never evaluated, so - // MySQLConstant.castAs does not support them; they must not be returned by getRandom(). - CHAR, DOUBLE; + // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never + // evaluated, so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + CHAR, FLOAT, DOUBLE, DECIMAL; public static CastType getRandom() { return SIGNED; diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index d16a8e517..5972996b8 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -80,8 +80,8 @@ protected MySQLExpression descend(MySQLExpression expr, boolean booleanContext) transformNode(op.getLeft(), SCALAR, false), transformNode(op.getRight(), SCALAR, false)); } else if (expr instanceof MySQLInOperation) { MySQLInOperation op = (MySQLInOperation) expr; - List listElements = op.getListElements().stream() - .map(e -> transformNode(e, SCALAR, false)).collect(Collectors.toList()); + List listElements = op.getListElements().stream().map(e -> transformNode(e, SCALAR, false)) + .collect(Collectors.toList()); return new MySQLInOperation(transformNode(op.getExpr(), SCALAR, false), listElements, op.isTrue()); } else if (expr instanceof MySQLComputableFunction) { MySQLComputableFunction op = (MySQLComputableFunction) expr; @@ -211,10 +211,18 @@ private CastType inferColumnType(MySQLColumnReference ref) { return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns case VARCHAR: return CastType.CHAR; + case FLOAT: + // Assumes FLOAT columns are never created with (M, D); otherwise the CAST would need the exact + // precision/scale. + return CastType.FLOAT; case DOUBLE: + // Assumes DOUBLE columns are never created with (M, D); otherwise the CAST would need the exact + // precision/scale. return CastType.DOUBLE; - case FLOAT: // FLOAT-to-DOUBLE widening in the CASE result changes the rendered value - case DECIMAL: // the CASE result would need the column's exact precision and scale + case DECIMAL: + // Assumes DECIMAL columns are never created with (M, D); otherwise the CAST would need the exact + // precision/scale. + return CastType.DECIMAL; default: return null; } @@ -247,8 +255,8 @@ private CastType inferCaseType(MySQLCaseOperator caseOp) { } /** - * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the - * same inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same + * inferrable type (a conservative under-approximation of MySQL's aggregation rules). */ private CastType commonType(MySQLExpression... exprs) { CastType common = null; From 4d6a651d5da612c26b4c180796b174c1fddc2dfc Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 23 Jul 2026 17:38:12 +0800 Subject: [PATCH 834/856] Fix treatment of unary minus in MySQL EET implementation --- .../mysql/oracle/MySQLEETTransformer.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 5972996b8..eb107c94b 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -185,13 +185,18 @@ private CastType inferUnaryPrefixType(MySQLUnaryPrefixOperation op) { } CastType operandType = inferType(op.getExpression()); if (op.getOp() == MySQLUnaryPrefixOperator.PLUS) { - return operandType; // unary + is the identity + return operandType; } - // Unary -: strings are coerced to DOUBLE; negating UNSIGNED changes the type (and usually errors). - if (operandType == CastType.CHAR || operandType == CastType.DOUBLE) { - return CastType.DOUBLE; + if (op.getOp() == MySQLUnaryPrefixOperator.MINUS) { + if (operandType == CastType.UNSIGNED) { + return CastType.SIGNED; + } else if (operandType == CastType.FLOAT) { + return CastType.DOUBLE; + } else if (operandType != CastType.CHAR) { + return operandType; + } } - return operandType == CastType.SIGNED ? CastType.SIGNED : null; + return null; } private CastType inferConstantType(MySQLConstant constant) { From e3d9f61ec8e09310aed7e8b686d777116c36776a Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 26 Jul 2026 15:02:27 +0800 Subject: [PATCH 835/856] Restrict creation of (M, D) columns to prevent false positives in EET --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 9 +++++++-- .../mysql/oracle/MySQLEETTransformer.java | 16 +++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index c17ccf0d5..40e325041 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -368,8 +368,13 @@ private void appendType(MySQLDataType randomType) { } } - public static void optionallyAddPrecisionAndScale(StringBuilder sb) { - if (Randomly.getBoolean() && !MySQLBugs.bug99183) { + private void optionallyAddPrecisionAndScale(StringBuilder sb) { + // The EET oracle's type inference assumes FLOAT/DOUBLE/DECIMAL columns are created without (M, D) (see + // MySQLEETTransformer#inferColumnType), so precision/scale is omitted while EET is active. This restriction can + // be lifted once (M, D) is tracked through the codebase and reflected in the CAST target types. + boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.EET); + if (Randomly.getBoolean() && !MySQLBugs.bug99183 && !eetActive) { sb.append("("); // The maximum number of digits (M) for DECIMAL is 65 long m = Randomly.getNotCachedInteger(1, 65); diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index eb107c94b..27a62869a 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -32,9 +32,10 @@ * nodes from their transformed children, threading the correct boolean/scalar context into each child. * *

- * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target types - * ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning {@code null} - * when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping it in a CAST. + * MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target + * types ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning + * {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping + * it in a CAST. */ public class MySQLEETTransformer extends EETTransformer { @@ -216,17 +217,14 @@ private CastType inferColumnType(MySQLColumnReference ref) { return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns case VARCHAR: return CastType.CHAR; + // FLOAT/DOUBLE/DECIMAL columns are created without (M, D) while EET is active, so the plain + // CAST target below matches the column's type. Reintroducing (M, D) for better coverage would + // require tracking it here and emitting the exact precision/scale in the CAST. case FLOAT: - // Assumes FLOAT columns are never created with (M, D); otherwise the CAST would need the exact - // precision/scale. return CastType.FLOAT; case DOUBLE: - // Assumes DOUBLE columns are never created with (M, D); otherwise the CAST would need the exact - // precision/scale. return CastType.DOUBLE; case DECIMAL: - // Assumes DECIMAL columns are never created with (M, D); otherwise the CAST would need the exact - // precision/scale. return CastType.DECIMAL; default: return null; From 0a09fba83235c0b129e10a11538e3835ee1a7a1c Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sun, 26 Jul 2026 15:15:33 +0800 Subject: [PATCH 836/856] Fix checkstyle violations --- src/sqlancer/common/gen/EETGenerator.java | 13 +++ src/sqlancer/common/oracle/EETOracle.java | 15 +++ .../common/oracle/EETTransformer.java | 110 ++++++++++++++++-- .../mysql/gen/MySQLExpressionGenerator.java | 2 +- .../mysql/oracle/MySQLEETTransformer.java | 7 +- 5 files changed, 135 insertions(+), 12 deletions(-) diff --git a/src/sqlancer/common/gen/EETGenerator.java b/src/sqlancer/common/gen/EETGenerator.java index ef436d975..a87a635ec 100644 --- a/src/sqlancer/common/gen/EETGenerator.java +++ b/src/sqlancer/common/gen/EETGenerator.java @@ -14,6 +14,17 @@ * Generator interface used by {@link sqlancer.common.oracle.EETOracle}. In addition to generating a random query (like * the other oracle generators), an EET generator creates a DBMS-specific {@link EETTransformer} that the oracle uses to * rewrite expressions into semantically equivalent ones. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class */ public interface EETGenerator, J extends Join, E extends Expression, T extends AbstractTable, C extends AbstractTableColumn> { @@ -33,6 +44,8 @@ public interface EETGenerator, J extends Join createTransformer(); } diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index 1422d5b6b..b5aedf2cb 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -26,6 +26,21 @@ * into semantically equivalent ones using {@link EETGenerator#transformExpression}. Because the transformation * preserves semantics, the original and the transformed query must return the same result set; any discrepancy * indicates a logic bug in the DBMS. + * + * @param + * the DBMS-specific SELECT statement class + * @param + * the DBMS-specific JOIN clause class + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific schema class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + * @param + * the DBMS-specific global state class */ public class EETOracle, J extends Join, E extends Expression, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> implements TestOracle { diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index 932a6fb8e..c46e148e1 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -24,7 +24,7 @@ public abstract class EETTransformer, T> { // true_expr(p) = p OR (NOT p) OR (p IS NULL) -> always TRUE private E trueExpr() { E p = generateBooleanExpression(); - return or(or(p, not(p)), isNull(p)); + return orExpr(orExpr(p, not(p)), isNull(p)); } // false_expr(p) = p AND (NOT p) AND (p IS NOT NULL) -> always FALSE @@ -40,6 +40,11 @@ private E falseExpr() { * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form of * rules No. 5 and 6). + * + * @param expr + * the expression whose static type the generated expression must match + * + * @return a random expression whose static type matches that of {@code expr} */ private E randExprOfSameType(E expr) { T type = inferType(expr); @@ -75,7 +80,7 @@ protected E applyRandomRule(E expr, boolean booleanContext) { } switch (rule) { case 1: // expr => false_expr OR expr - return or(falseExpr(), expr); + return orExpr(falseExpr(), expr); case 2: // expr => true_expr AND expr return and(trueExpr(), expr); case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END @@ -94,6 +99,13 @@ protected E applyRandomRule(E expr, boolean booleanContext) { /** * Transforms {@code expr} into a semantically equivalent expression. A transformation rule is always applied at the * root, guaranteeing (unless only rule 7 is applicable) that the returned expression differs from the input. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * + * @return a semantically equivalent expression */ public E transform(E expr, boolean booleanContext) { return transformNode(expr, booleanContext, true); @@ -101,6 +113,15 @@ public E transform(E expr, boolean booleanContext) { /** * Descends into {@code expr}, rebuilds it from transformed children, then optionally applies a rule at this node. + * + * @param expr + * the expression to transform + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value + * @param forceApply + * whether a rule must be applied at this node rather than only with some probability + * + * @return the transformed expression */ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { E descended = descend(expr, booleanContext); @@ -124,25 +145,79 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { */ protected abstract E descend(E expr, boolean booleanContext); - /** Builds {@code left AND right}. */ + /** + * Builds {@code left AND right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left AND right} expression + */ protected abstract E and(E left, E right); - /** Builds {@code left OR right}. */ - protected abstract E or(E left, E right); + /** + * Builds {@code left OR right}. + * + * @param left + * the left operand + * @param right + * the right operand + * + * @return the {@code left OR right} expression + */ + protected abstract E orExpr(E left, E right); - /** Builds {@code NOT expr}. */ + /** + * Builds {@code NOT expr}. + * + * @param expr + * the operand + * + * @return the {@code NOT expr} expression + */ protected abstract E not(E expr); - /** Builds {@code expr IS NULL}. */ + /** + * Builds {@code expr IS NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NULL} expression + */ protected abstract E isNull(E expr); - /** Builds {@code expr IS NOT NULL}. */ + /** + * Builds {@code expr IS NOT NULL}. + * + * @param expr + * the operand + * + * @return the {@code expr IS NOT NULL} expression + */ protected abstract E isNotNull(E expr); - /** Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. */ + /** + * Builds {@code CASE WHEN condition THEN thenExpr ELSE elseExpr END}. + * + * @param condition + * the WHEN condition + * @param thenExpr + * the THEN branch + * @param elseExpr + * the ELSE branch + * + * @return the CASE WHEN expression + */ protected abstract E caseWhen(E condition, E thenExpr, E elseExpr); - /** Generates a fresh random boolean expression, reusing the variables available to the query generator. */ + /** + * Generates a fresh random boolean expression, reusing the variables available to the query generator. + * + * @return a fresh random boolean expression + */ protected abstract E generateBooleanExpression(); /** @@ -152,6 +227,11 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * {@code null} is always safe — rules No. 3 and 4 then fall back to reusing {@code expr} itself as the dead branch. * Inference should therefore be conservative: prefer {@code null} over a type whose CASE WHEN behaviour is * uncertain. + * + * @param expr + * the expression whose static type is to be inferred + * + * @return the inferred static type of {@code expr}, or {@code null} if it cannot be determined */ protected abstract T inferType(E expr); @@ -160,12 +240,22 @@ protected E transformNode(E expr, boolean booleanContext, boolean forceApply) { * generator. DBMSs with a typed expression generator can delegate to it directly; DBMSs with an untyped generator * can instead wrap an arbitrary random expression in a CAST to {@code type} (which requires every value of * {@code T} to be a valid CAST target). + * + * @param type + * the static type the generated expression must have + * + * @return a fresh random expression of static type {@code type} */ protected abstract E generateExpressionOfType(T type); /** * Whether {@code expr} may be wrapped in a CASE WHEN expression. Some expressions (e.g. table references) are not * CASE-WHEN applicable and must be transformed to themselves (rule No. 7 of the EET paper). + * + * @param expr + * the expression to test + * + * @return {@code true} if {@code expr} may be wrapped in a CASE WHEN expression */ protected abstract boolean isCaseWhenApplicable(E expr); } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9b019a6d4..da304ac67 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -12,6 +12,7 @@ import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; +import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTables; import sqlancer.mysql.MySQLBugs; import sqlancer.mysql.MySQLGlobalState; @@ -46,7 +47,6 @@ import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation; import sqlancer.mysql.ast.MySQLUnaryPrefixOperation.MySQLUnaryPrefixOperator; -import sqlancer.common.oracle.EETTransformer; import sqlancer.mysql.oracle.MySQLEETTransformer; public class MySQLExpressionGenerator extends UntypedExpressionGenerator diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 27a62869a..2e4394eec 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -118,7 +118,7 @@ protected MySQLExpression and(MySQLExpression left, MySQLExpression right) { } @Override - protected MySQLExpression or(MySQLExpression left, MySQLExpression right) { + protected MySQLExpression orExpr(MySQLExpression left, MySQLExpression right) { return new MySQLBinaryLogicalOperation(left, right, MySQLBinaryLogicalOperator.OR); } @@ -260,6 +260,11 @@ private CastType inferCaseType(MySQLCaseOperator caseOp) { /** * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same * inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * + * @param exprs + * the result-type-determining subexpressions + * + * @return the common inferred type of {@code exprs}, or {@code null} if they do not share one */ private CastType commonType(MySQLExpression... exprs) { CastType common = null; From c71d198fff6cb271e1f946addbe5c43597580636 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 13:09:02 +0800 Subject: [PATCH 837/856] Refactor EETTransformer to encode the transformation rules as an enum --- .../common/oracle/EETTransformer.java | 159 ++++++++++++++---- 1 file changed, 129 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/common/oracle/EETTransformer.java b/src/sqlancer/common/oracle/EETTransformer.java index c46e148e1..b472b8aff 100644 --- a/src/sqlancer/common/oracle/EETTransformer.java +++ b/src/sqlancer/common/oracle/EETTransformer.java @@ -1,5 +1,8 @@ package sqlancer.common.oracle; +import java.util.ArrayList; +import java.util.List; + import sqlancer.Randomly; import sqlancer.common.ast.newast.Expression; @@ -35,11 +38,10 @@ private E falseExpr() { /** * Implements the paper's {@code rand_expr(type(expr))}: a random expression whose static type matches that of - * {@code expr}. Although the generated expression is never evaluated (it occupies the dead branch of rules No. 3 + * {@code expr}. Although the generated expression is never evaluated (it occupies the redundant branch of rules 3 * and 4), its static type participates in the DBMS's CASE WHEN result-type resolution, so a type mismatch could * alter the live branch's value or rendering. When the type of {@code expr} cannot be inferred, this falls back to - * {@code expr} itself, which trivially has the correct type (degenerating the rule to the {@code copy_expr} form of - * rules No. 5 and 6). + * {@code expr} itself, which trivially has the correct type (degenerating to rules 5 and 6). * * @param expr * the expression whose static type the generated expression must match @@ -54,46 +56,143 @@ private E randExprOfSameType(E expr) { return generateExpressionOfType(type); } + /** + * The first six transformation rules of the EET paper (Table 2). Each rule knows how to apply itself + * ({@link #apply}) and in which contexts it preserves the expression's value ({@link #isApplicable}). Rule No. 7 + * (transform the expression to itself) is not modelled here: it is the fallback applied by {@link #applyRandomRule} + * when no other rule is applicable. + */ + private enum Rule { + // expr => false_expr OR expr + RULE_1 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.orExpr(t.falseExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => true_expr AND expr + RULE_2 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.and(t.trueExpr(), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + // Reduces the expression to a boolean value, so it is value-preserving only in a boolean context. + return booleanContext; + } + }, + // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END + RULE_3 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.falseExpr(), t.randExprOfSameType(expr), expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END + RULE_4 { + @Override + , T> E apply(EETTransformer t, E expr) { + return t.caseWhen(t.trueExpr(), expr, t.randExprOfSameType(expr)); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END + RULE_5 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }, + // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END + RULE_6 { + @Override + , T> E apply(EETTransformer t, E expr) { + // deep copy of expr is not needed, as the AST nodes are immutable anyway + return t.caseWhen(t.generateBooleanExpression(), expr, expr); + } + + @Override + boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable) { + return caseWhenApplicable; + } + }; + + /** + * Applies this rule to {@code expr}, producing a semantically equivalent expression. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific type domain + * @param t + * the transformer providing the DBMS-specific node factories + * @param expr + * the expression to transform + * + * @return a semantically equivalent expression + */ + abstract , T> E apply(EETTransformer t, E expr); + + /** + * Whether this rule preserves {@code expr}'s value in the given context. + * + * @param booleanContext + * whether {@code expr} is evaluated purely for its truth value (rules 1 and 2 are only applicable if + * this is the case) + * @param caseWhenApplicable + * whether {@code expr} may be wrapped in a CASE WHEN expression + * + * @return {@code true} if this rule preserves {@code expr}'s value in the given context + */ + abstract boolean isApplicable(boolean booleanContext, boolean caseWhenApplicable); + } + /** * Applies a randomly chosen applicable transformation rule to {@code expr}, returning a semantically equivalent - * expression. + * expression. When no rule is applicable, {@code expr} is returned unchanged (rule No. 7 of the EET paper). * * @param expr * the expression to transform * @param booleanContext - * whether {@code expr} is evaluated purely for its truth value; only in a boolean context may the - * determined-boolean rules (No. 1 and 2), which reduce the expression to a boolean value, be applied + * whether {@code expr} is evaluated purely for its truth value * * @return a semantically equivalent expression */ protected E applyRandomRule(E expr, boolean booleanContext) { - int rule; - if (booleanContext) { - // Rules No. 1-6 are all value-preserving in a boolean context. - rule = Randomly.fromOptions(1, 2, 3, 4, 5, 6); - } else { - if (!isCaseWhenApplicable(expr)) { - return expr; // rule No. 7: transform the expression to itself + boolean caseWhenApplicable = isCaseWhenApplicable(expr); + List applicableRules = new ArrayList<>(); + for (Rule rule : Rule.values()) { + if (rule.isApplicable(booleanContext, caseWhenApplicable)) { + applicableRules.add(rule); } - // In a scalar context only the CASE WHEN rules preserve the exact value and type. - rule = Randomly.fromOptions(3, 4, 5, 6); } - switch (rule) { - case 1: // expr => false_expr OR expr - return orExpr(falseExpr(), expr); - case 2: // expr => true_expr AND expr - return and(trueExpr(), expr); - case 3: // expr => CASE WHEN false_expr THEN rand_expr(type(expr)) ELSE expr END - return caseWhen(falseExpr(), randExprOfSameType(expr), expr); - case 4: // expr => CASE WHEN true_expr THEN expr ELSE rand_expr(type(expr)) END - return caseWhen(trueExpr(), expr, randExprOfSameType(expr)); - case 5: // expr => CASE WHEN rand_expr(boolean) THEN copy(expr) ELSE expr END - case 6: // expr => CASE WHEN rand_expr(boolean) THEN expr ELSE copy(expr) END - return caseWhen(generateBooleanExpression(), expr, expr); - // deep copy of expr is not needed, as the AST nodes are immutable anyway - default: - throw new AssertionError(rule); + if (applicableRules.isEmpty()) { + return expr; // rule 7 fallback: transform expression to itself } + return Randomly.fromList(applicableRules).apply(this, expr); } /** From 29291c7a7c73eafd2621240bec3da4d8db4be6dd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 14:35:44 +0800 Subject: [PATCH 838/856] Factor out reproducer logic for reuse among multiple different test oracles --- .../oracle/AbstractComparisonReproducer.java | 142 ++++++++++++++++++ src/sqlancer/common/oracle/EETOracle.java | 72 ++++----- src/sqlancer/common/oracle/NoRECOracle.java | 62 ++++---- .../common/oracle/TLPWhereOracle.java | 85 ++++++----- 4 files changed, 239 insertions(+), 122 deletions(-) create mode 100644 src/sqlancer/common/oracle/AbstractComparisonReproducer.java diff --git a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java new file mode 100644 index 000000000..d20a9636a --- /dev/null +++ b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java @@ -0,0 +1,142 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; + +/** + * Shared skeleton for the {@link Reproducer}s of oracles that detect a bug by comparing two evaluations of a + * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}. All of these + * reduce the bug the same way: re-evaluate both sides against the reduced database and report whether they still + * disagree (or, when the original bug was an unexpected DBMS error, whether that same error still fires). + * + *

+ * This class owns that control flow (including distinguishing a still-reproducing error from an unrelated one + * introduced by the reduction) and the {@link #getBugInformation()} header. Subclasses supply the parts specific to + * their oracle: how each side is evaluated, how the two are compared, and how the failing queries are rendered in the + * reduced test case. + * + * @param + * the DBMS-specific global state class + * @param + * the type each side evaluates to (e.g. a result set as a list of strings, a row count, a post-image) + */ +public abstract class AbstractComparisonReproducer, R> implements Reproducer { + + /** + * The message of the unexpected DBMS error the original bug was, or {@code null} if the original bug was a + * comparison mismatch rather than an error. + */ + protected final String expectedErrorMessage; + + protected AbstractComparisonReproducer(String expectedErrorMessage) { + this.expectedErrorMessage = expectedErrorMessage; + } + + /** + * Whether the recorded bug has a transformed (second) side. It does not when the bug was a DBMS error triggered by + * the original query alone, in which case there is no second side to evaluate or compare. + * + * @return {@code true} if {@link #evaluateTransformed} should be called + */ + protected abstract boolean hasTransformedSide(); + + /** + * Evaluates the original side against the (reduced) database. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @return the original side's value + * + * @throws SQLException + * if a DBMS interaction fails + */ + protected abstract R evaluateOriginal(G globalState) throws SQLException; + + /** + * Evaluates the transformed side against the (reduced) database. Only called when {@link #hasTransformedSide()} is + * {@code true}. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @return the transformed side's value + * + * @throws SQLException + * if a DBMS interaction fails + */ + protected abstract R evaluateTransformed(G globalState) throws SQLException; + + /** + * Whether the two evaluated sides disagree in the way that constitutes the bug. + * + * @param original + * the original side's value + * @param transformed + * the transformed side's value + * @param globalState + * the state the sides were evaluated against + * + * @return {@code true} if the sides differ (i.e. the bug still triggers) + */ + protected abstract boolean sidesDiffer(R original, R transformed, G globalState); + + @Override + public final boolean bugStillTriggers(G globalState) { + R original; + R transformed; + try { + original = evaluateOriginal(globalState); + if (!hasTransformedSide()) { + // the original bug was a DBMS error on the original query alone, which no longer occurs + return false; + } + transformed = evaluateTransformed(globalState); + } catch (AssertionError unexpectedError) { + // a DBMS error reproduces the bug only if the original failure was the same error; + // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) + return expectedErrorMessage != null + && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + if (expectedErrorMessage != null) { + // the original bug was a DBMS error, which no longer occurs + return false; + } + return sidesDiffer(original, transformed, globalState); + } + + @Override + public final String getBugInformation() { + StringBuilder sb = new StringBuilder(); + if (expectedErrorMessage != null) { + sb.append("-- On the database set up by the statements above, the following queries trigger an" + + " unexpected error with message: ").append(expectedErrorMessage).append(System.lineSeparator()); + } else { + sb.append(mismatchHeaderLine()).append(System.lineSeparator()); + } + appendQueryLines(sb); + return sb.toString(); + } + + /** + * The header line (without trailing line separator) describing the mismatch, used when the original bug was a + * comparison mismatch rather than an error. For example, "-- On the database set up by the statements above, the + * result sets of the following queries mismatch:". + * + * @return the mismatch header line + */ + protected abstract String mismatchHeaderLine(); + + /** + * Appends the failing queries (or statements) to {@code sb}, one commented line each, so the reduced test case is + * self-contained. Called for both the mismatch and the error case, after the header. + * + * @param sb + * the builder to append to + */ + protected abstract void appendQueryLines(StringBuilder sb); +} diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index b5aedf2cb..c3ad4bae9 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -53,50 +53,38 @@ public class EETOracle, J extends Join, E private Reproducer reproducer; private String generatedQueryString; - private final class EETReproducer implements Reproducer { + private final class EETReproducer extends AbstractComparisonReproducer> { private final String originalQueryString; // null if the original bug was a DBMS error on the original query alone private final String transformedQueryString; - // null if the original bug is a result set mismatch; otherwise, the message of the - // unexpected DBMS error that the original or transformed query triggered - private final String expectedErrorMessage; EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { + super(expectedErrorMessage); this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; - this.expectedErrorMessage = expectedErrorMessage; } @Override - public boolean bugStillTriggers(G globalState) { - List originalResultSet; - List transformedResultSet; - try { - // Re-execute both queries against the current (reduced) database instead of comparing - // against a cached result set, which would be stale once statements have been removed. - originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - globalState); - if (transformedQueryString == null) { - // the original bug was a DBMS error on the original query alone, which no - // longer occurs - return false; - } - transformedResultSet = ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, - globalState); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (SQLException | RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs - return false; - } + protected boolean hasTransformedSide() { + return transformedQueryString != null; + } + + @Override + protected List evaluateOriginal(G globalState) throws SQLException { + // Re-execute against the current (reduced) database instead of comparing against a cached result set, + // which would be stale once statements have been removed. + return ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + } + + @Override + protected List evaluateTransformed(G globalState) throws SQLException { + return ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); + } + + @Override + protected boolean sidesDiffer(List original, List transformed, G globalState) { try { - ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, + ComparatorHelper.assumeResultSetsAreEqual(original, transformed, originalQueryString, List.of(transformedQueryString), globalState); } catch (AssertionError resultSetMismatch) { return true; @@ -105,21 +93,17 @@ public boolean bugStillTriggers(G globalState) { } @Override - public String getBugInformation() { - StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage == null) { - sb.append("-- On the database set up by the statements above, the result sets of the following" - + " queries mismatch:").append(System.lineSeparator()); - } else { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage) - .append(System.lineSeparator()); - } + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); if (transformedQueryString != null) { sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); } - return sb.toString(); } } diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 35f0c26af..11e52b91d 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -30,63 +30,55 @@ public class NoRECOracle, J extends Join, private Reproducer reproducer; private String lastQueryString; - private static class NoRECReproducer> implements Reproducer { + private static class NoRECReproducer> + extends AbstractComparisonReproducer { private final Function optimizedQuery; private final Function unoptimizedQuery; private final String optimizedQueryString; private final String unoptimizedQueryString; - // null if the original bug is a count mismatch; otherwise, the message of the unexpected - // DBMS error that the original queries triggered - private final String expectedErrorMessage; NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, String optimizedQueryString, String unoptimizedQueryString, String expectedErrorMessage) { + super(expectedErrorMessage); this.optimizedQuery = optimizedQuery; this.unoptimizedQuery = unoptimizedQuery; this.optimizedQueryString = optimizedQueryString; this.unoptimizedQueryString = unoptimizedQueryString; - this.expectedErrorMessage = expectedErrorMessage; } @Override - public boolean bugStillTriggers(G globalState) { - int optimizedCount; - int unoptimizedCount; - try { - optimizedCount = optimizedQuery.apply(globalState); - unoptimizedCount = unoptimizedQuery.apply(globalState); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs - return false; - } + protected boolean hasTransformedSide() { + return true; + } + + @Override + protected Integer evaluateOriginal(G globalState) { + return optimizedQuery.apply(globalState); + } + + @Override + protected Integer evaluateTransformed(G globalState) { + return unoptimizedQuery.apply(globalState); + } + + @Override + protected boolean sidesDiffer(Integer optimizedCount, Integer unoptimizedCount, G globalState) { if (optimizedCount == -1 || unoptimizedCount == -1) { return false; } - return optimizedCount != unoptimizedCount; + return optimizedCount.intValue() != unoptimizedCount.intValue(); } @Override - public String getBugInformation() { - StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage == null) { - sb.append("-- On the database set up by the statements above, the row counts of the following" - + " queries mismatch:").append(System.lineSeparator()); - } else { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage) - .append(System.lineSeparator()); - } + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the row counts of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); - return sb.toString(); } } diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index 8fc87ac3d..bf759ebfc 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -29,56 +29,59 @@ public class TLPWhereOracle, J extends Join reproducer; private String generatedQueryString; - private class TLPWhereReproducer implements Reproducer { + // A side's result set, together with the human-readable combined-query strings that + // getCombinedResultSet fills in for the transformed side (unused, and null, for the original side) + private static final class TLPResultSet { + final List resultSet; + final List combinedString; + + TLPResultSet(List resultSet, List combinedString) { + this.resultSet = resultSet; + this.combinedString = combinedString; + } + } + + private class TLPWhereReproducer extends AbstractComparisonReproducer { final String firstQueryString; final String secondQueryString; final String thirdQueryString; final String originalQueryString; final boolean orderBy; - // null if the original bug is a result set mismatch; otherwise, the message of the - // unexpected DBMS error that the original queries triggered - final String expectedErrorMessage; TLPWhereReproducer(String firstQueryString, String secondQueryString, String thirdQueryString, String originalQueryString, boolean orderBy, String expectedErrorMessage) { + super(expectedErrorMessage); this.firstQueryString = firstQueryString; this.secondQueryString = secondQueryString; this.thirdQueryString = thirdQueryString; this.originalQueryString = originalQueryString; this.orderBy = orderBy; - this.expectedErrorMessage = expectedErrorMessage; } @Override - public boolean bugStillTriggers(G globalState) { - List firstResultSet; + protected boolean hasTransformedSide() { + return firstQueryString != null; + } + + @Override + protected TLPResultSet evaluateOriginal(G globalState) throws SQLException { + return new TLPResultSet( + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState), null); + } + + @Override + protected TLPResultSet evaluateTransformed(G globalState) throws SQLException { List combinedString = new ArrayList<>(); - List secondResultSet; - try { - firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, - globalState); - if (firstQueryString == null) { - // the original bug was a DBMS error on the original query alone, which no - // longer occurs - return false; - } - secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, - thirdQueryString, combinedString, !orderBy, globalState, errors); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (SQLException | RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs - return false; - } + List secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, + thirdQueryString, combinedString, !orderBy, globalState, errors); + return new TLPResultSet(secondResultSet, combinedString); + } + + @Override + protected boolean sidesDiffer(TLPResultSet original, TLPResultSet transformed, G globalState) { try { - ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, - combinedString, globalState); + ComparatorHelper.assumeResultSetsAreEqual(original.resultSet, transformed.resultSet, + originalQueryString, transformed.combinedString, globalState); } catch (AssertionError resultSetMismatch) { return true; } @@ -86,16 +89,13 @@ public boolean bugStillTriggers(G globalState) { } @Override - public String getBugInformation() { - StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage == null) { - sb.append("-- On the database set up by the statements above, the result sets of the following" - + " queries mismatch:").append(System.lineSeparator()); - } else { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage) - .append(System.lineSeparator()); - } + protected String mismatchHeaderLine() { + return "-- On the database set up by the statements above, the result sets of the following" + + " queries mismatch:"; + } + + @Override + protected void appendQueryLines(StringBuilder sb) { sb.append("-- ").append(originalQueryString).append(';').append(System.lineSeparator()); if (firstQueryString != null) { if (orderBy) { @@ -107,7 +107,6 @@ public String getBugInformation() { .append(" UNION ALL ").append(thirdQueryString).append(';').append(System.lineSeparator()); } } - return sb.toString(); } } From 2a74be3f623f67398c96e31a4e4f75869f18e973 Mon Sep 17 00:00:00 2001 From: splf Date: Thu, 30 Jul 2026 16:08:07 +0500 Subject: [PATCH 839/856] Do not schedule disabled PostgreSQL tablespaces --- src/sqlancer/postgres/PostgresProvider.java | 2 +- .../postgres/TestPostgresProvider.java | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 test/sqlancer/postgres/TestPostgresProvider.java diff --git a/src/sqlancer/postgres/PostgresProvider.java b/src/sqlancer/postgres/PostgresProvider.java index ec7978216..814013df6 100644 --- a/src/sqlancer/postgres/PostgresProvider.java +++ b/src/sqlancer/postgres/PostgresProvider.java @@ -194,7 +194,7 @@ protected static int mapActions(PostgresGlobalState globalState, Action a) { nrPerformed = r.getInteger(0, 2); break; case CREATE_TABLESPACE: - nrPerformed = r.getInteger(0, 2); + nrPerformed = globalState.getDbmsSpecificOptions().isTestTablespaces() ? r.getInteger(0, 2) : 0; break; case UPDATE: nrPerformed = r.getInteger(0, 10); diff --git a/test/sqlancer/postgres/TestPostgresProvider.java b/test/sqlancer/postgres/TestPostgresProvider.java new file mode 100644 index 000000000..81e795568 --- /dev/null +++ b/test/sqlancer/postgres/TestPostgresProvider.java @@ -0,0 +1,21 @@ +package sqlancer.postgres; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import org.junit.jupiter.api.Test; + +class TestPostgresProvider { + + @Test + void createTablespaceIsNotScheduledWhenDisabled() { + PostgresGlobalState state = new PostgresGlobalState(); + state.setDbmsSpecificOptions(new PostgresOptions() { + @Override + public boolean isTestTablespaces() { + return false; + } + }); + + assertEquals(0, PostgresProvider.mapActions(state, PostgresProvider.Action.CREATE_TABLESPACE)); + } +} From e954c10c75c79ac11f4bbf52c2edd00a6d442fdd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 20:22:31 +0800 Subject: [PATCH 840/856] Factor out unexpected-error reproduction into dedicated UnexpectedErrorReproducer --- .../oracle/AbstractComparisonReproducer.java | 73 +++++------------ src/sqlancer/common/oracle/EETOracle.java | 44 +++++++---- src/sqlancer/common/oracle/NoRECOracle.java | 36 ++++++--- .../common/oracle/TLPWhereOracle.java | 60 +++++++++----- .../oracle/UnexpectedErrorReproducer.java | 79 +++++++++++++++++++ 5 files changed, 190 insertions(+), 102 deletions(-) create mode 100644 src/sqlancer/common/oracle/UnexpectedErrorReproducer.java diff --git a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java index d20a9636a..582a19c1d 100644 --- a/src/sqlancer/common/oracle/AbstractComparisonReproducer.java +++ b/src/sqlancer/common/oracle/AbstractComparisonReproducer.java @@ -7,43 +7,28 @@ /** * Shared skeleton for the {@link Reproducer}s of oracles that detect a bug by comparing two evaluations of a - * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}. All of these - * reduce the bug the same way: re-evaluate both sides against the reduced database and report whether they still - * disagree (or, when the original bug was an unexpected DBMS error, whether that same error still fires). + * semantically-equivalent pair (e.g. {@link EETOracle}, {@link NoRECOracle}, {@link TLPWhereOracle}). Reduction re-runs + * both sides against the reduced database and reports whether they still disagree. * *

- * This class owns that control flow (including distinguishing a still-reproducing error from an unrelated one - * introduced by the reduction) and the {@link #getBugInformation()} header. Subclasses supply the parts specific to - * their oracle: how each side is evaluated, how the two are compared, and how the failing queries are rendered in the - * reduced test case. + * The separate case where the original bug was an unexpected DBMS error rather than a mismatch is handled by + * {@link UnexpectedErrorReproducer}, so a subclass here deals only with comparing two sides and never with error + * handling. + * + *

+ * This class owns the compare-and-report control flow and the {@link #getBugInformation()} header. Subclasses supply + * how each side is evaluated, how the two are compared, and how the failing queries are rendered in the reduced test + * case. * * @param * the DBMS-specific global state class * @param - * the type each side evaluates to (e.g. a result set as a list of strings, a row count, a post-image) + * the type each side evaluates to (e.g. a result set as a list of strings, a row count) */ public abstract class AbstractComparisonReproducer, R> implements Reproducer { /** - * The message of the unexpected DBMS error the original bug was, or {@code null} if the original bug was a - * comparison mismatch rather than an error. - */ - protected final String expectedErrorMessage; - - protected AbstractComparisonReproducer(String expectedErrorMessage) { - this.expectedErrorMessage = expectedErrorMessage; - } - - /** - * Whether the recorded bug has a transformed (second) side. It does not when the bug was a DBMS error triggered by - * the original query alone, in which case there is no second side to evaluate or compare. - * - * @return {@code true} if {@link #evaluateTransformed} should be called - */ - protected abstract boolean hasTransformedSide(); - - /** - * Evaluates the original side against the (reduced) database. + * Evaluates the original side against the reduced database. * * @param globalState * the state whose connection points at the reduced database @@ -56,8 +41,7 @@ protected AbstractComparisonReproducer(String expectedErrorMessage) { protected abstract R evaluateOriginal(G globalState) throws SQLException; /** - * Evaluates the transformed side against the (reduced) database. Only called when {@link #hasTransformedSide()} is - * {@code true}. + * Evaluates the transformed side against the reduced database. * * @param globalState * the state whose connection points at the reduced database @@ -89,21 +73,9 @@ public final boolean bugStillTriggers(G globalState) { R transformed; try { original = evaluateOriginal(globalState); - if (!hasTransformedSide()) { - // the original bug was a DBMS error on the original query alone, which no longer occurs - return false; - } transformed = evaluateTransformed(globalState); - } catch (AssertionError unexpectedError) { - // a DBMS error reproduces the bug only if the original failure was the same error; - // other errors are artifacts of the reduction (e.g., a removed CREATE TABLE) - return expectedErrorMessage != null - && expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); - } catch (SQLException | RuntimeException e) { - return false; - } - if (expectedErrorMessage != null) { - // the original bug was a DBMS error, which no longer occurs + } catch (AssertionError | SQLException | RuntimeException e) { + // any failure re-running the two sides means this reduced database no longer shows the mismatch return false; } return sidesDiffer(original, transformed, globalState); @@ -112,28 +84,21 @@ public final boolean bugStillTriggers(G globalState) { @Override public final String getBugInformation() { StringBuilder sb = new StringBuilder(); - if (expectedErrorMessage != null) { - sb.append("-- On the database set up by the statements above, the following queries trigger an" - + " unexpected error with message: ").append(expectedErrorMessage).append(System.lineSeparator()); - } else { - sb.append(mismatchHeaderLine()).append(System.lineSeparator()); - } + sb.append(mismatchHeaderLine()).append(System.lineSeparator()); appendQueryLines(sb); return sb.toString(); } /** - * The header line (without trailing line separator) describing the mismatch, used when the original bug was a - * comparison mismatch rather than an error. For example, "-- On the database set up by the statements above, the - * result sets of the following queries mismatch:". + * The header line (without trailing line separator) describing the mismatch. For example, "-- On the database set + * up by the statements above, the result sets of the following queries mismatch:". * * @return the mismatch header line */ protected abstract String mismatchHeaderLine(); /** - * Appends the failing queries (or statements) to {@code sb}, one commented line each, so the reduced test case is - * self-contained. Called for both the mismatch and the error case, after the header. + * Appends the failing queries to {@code sb}, one commented line each, so the reduced test case is self-contained. * * @param sb * the builder to append to diff --git a/src/sqlancer/common/oracle/EETOracle.java b/src/sqlancer/common/oracle/EETOracle.java index c3ad4bae9..0cd4ae99e 100644 --- a/src/sqlancer/common/oracle/EETOracle.java +++ b/src/sqlancer/common/oracle/EETOracle.java @@ -55,20 +55,13 @@ public class EETOracle, J extends Join, E private final class EETReproducer extends AbstractComparisonReproducer> { private final String originalQueryString; - // null if the original bug was a DBMS error on the original query alone private final String transformedQueryString; - EETReproducer(String originalQueryString, String transformedQueryString, String expectedErrorMessage) { - super(expectedErrorMessage); + EETReproducer(String originalQueryString, String transformedQueryString) { this.originalQueryString = originalQueryString; this.transformedQueryString = transformedQueryString; } - @Override - protected boolean hasTransformedSide() { - return transformedQueryString != null; - } - @Override protected List evaluateOriginal(G globalState) throws SQLException { // Re-execute against the current (reduced) database instead of comparing against a cached result set, @@ -100,11 +93,32 @@ protected String mismatchHeaderLine() { @Override protected void appendQueryLines(StringBuilder sb) { - sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + renderQueryLines(sb, originalQueryString, transformedQueryString); + } + } + + // Renders the failing queries as commented lines, shared by the mismatch and the unexpected-error reproducers. + // transformedQueryString is null when the error struck the original query before any transformation existed. + private static void renderQueryLines(StringBuilder sb, String originalQueryString, String transformedQueryString) { + sb.append("-- original: ").append(originalQueryString).append(';').append(System.lineSeparator()); + if (transformedQueryString != null) { + sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); + } + } + + // Builds the reproducer for an unexpected DBMS error, which re-runs the query (or both queries) and checks the same + // error still fires. transformedQueryString is null when only the original query ran before the error. + private UnexpectedErrorReproducer errorReproducer(String originalQueryString, String transformedQueryString, + String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); if (transformedQueryString != null) { - sb.append("-- transformed: ").append(transformedQueryString).append(';').append(System.lineSeparator()); + ComparatorHelper.getResultSetFirstColumnAsString(transformedQueryString, errors, globalState); } - } + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, originalQueryString, transformedQueryString); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); } public EETOracle(G state, EETGenerator gen, ExpectedErrors expectedErrors) { @@ -139,8 +153,8 @@ public void check() throws SQLException { originalResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); } catch (AssertionError unexpectedError) { // an unexpected DBMS error on the original query alone is itself a bug worth reducing; - // transformedQueryString is null because no transformed query is involved - reproducer = new EETReproducer(originalQueryString, null, + // there is no transformed query yet, so only the original is replayed + reproducer = errorReproducer(originalQueryString, null, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } @@ -160,14 +174,14 @@ public void check() throws SQLException { } catch (AssertionError unexpectedError) { // the semantics-preserving transformation made the query trigger a DBMS error that the // original did not, which is a bug worth reducing - reproducer = new EETReproducer(originalQueryString, transformedQueryString, + reproducer = errorReproducer(originalQueryString, transformedQueryString, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } // Set the reproducer before the assertion: assumeResultSetsAreEqual throws when the bug is // detected, so creating the reproducer afterwards would leave it null and prevent any reduction. - reproducer = new EETReproducer(originalQueryString, transformedQueryString, null); + reproducer = new EETReproducer(originalQueryString, transformedQueryString); ComparatorHelper.assumeResultSetsAreEqual(originalResultSet, transformedResultSet, originalQueryString, List.of(transformedQueryString), state); diff --git a/src/sqlancer/common/oracle/NoRECOracle.java b/src/sqlancer/common/oracle/NoRECOracle.java index 11e52b91d..ba48e80bb 100644 --- a/src/sqlancer/common/oracle/NoRECOracle.java +++ b/src/sqlancer/common/oracle/NoRECOracle.java @@ -38,19 +38,13 @@ private static class NoRECReproducer> private final String unoptimizedQueryString; NoRECReproducer(Function optimizedQuery, Function unoptimizedQuery, - String optimizedQueryString, String unoptimizedQueryString, String expectedErrorMessage) { - super(expectedErrorMessage); + String optimizedQueryString, String unoptimizedQueryString) { this.optimizedQuery = optimizedQuery; this.unoptimizedQuery = unoptimizedQuery; this.optimizedQueryString = optimizedQueryString; this.unoptimizedQueryString = unoptimizedQueryString; } - @Override - protected boolean hasTransformedSide() { - return true; - } - @Override protected Integer evaluateOriginal(G globalState) { return optimizedQuery.apply(globalState); @@ -77,11 +71,29 @@ protected String mismatchHeaderLine() { @Override protected void appendQueryLines(StringBuilder sb) { - sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); - sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + renderQueryLines(sb, optimizedQueryString, unoptimizedQueryString); } } + // Renders the failing queries as commented lines, shared by the mismatch and the unexpected-error reproducers. + private static void renderQueryLines(StringBuilder sb, String optimizedQueryString, String unoptimizedQueryString) { + sb.append("-- optimized: ").append(optimizedQueryString).append(';').append(System.lineSeparator()); + sb.append("-- unoptimized: ").append(unoptimizedQueryString).append(';').append(System.lineSeparator()); + } + + // Builds the reproducer for an unexpected DBMS error, which re-runs both queries and checks the same error fires. + private static > UnexpectedErrorReproducer errorReproducer( + Function optimizedQuery, Function unoptimizedQuery, String optimizedQueryString, + String unoptimizedQueryString, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + optimizedQuery.apply(globalState); + unoptimizedQuery.apply(globalState); + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, optimizedQueryString, unoptimizedQueryString); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + public NoRECOracle(G state, NoRECGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -128,8 +140,8 @@ public void check() throws SQLException { optimizedCount = optimizedQuery.apply(state); unoptimizedCount = unoptimizedQuery.apply(state); } catch (AssertionError unexpectedError) { - reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, - unoptimizedQueryString, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + reproducer = errorReproducer(optimizedQuery, unoptimizedQuery, optimizedQueryString, unoptimizedQueryString, + TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } @@ -139,7 +151,7 @@ public void check() throws SQLException { if (unoptimizedCount != optimizedCount) { reproducer = new NoRECReproducer<>(optimizedQuery, unoptimizedQuery, optimizedQueryString, - unoptimizedQueryString, null); + unoptimizedQueryString); String queryFormatString = "-- %s;\n-- count: %d"; String firstQueryStringWithCount = String.format(queryFormatString, optimizedQueryString, optimizedCount); diff --git a/src/sqlancer/common/oracle/TLPWhereOracle.java b/src/sqlancer/common/oracle/TLPWhereOracle.java index bf759ebfc..5e1d2861b 100644 --- a/src/sqlancer/common/oracle/TLPWhereOracle.java +++ b/src/sqlancer/common/oracle/TLPWhereOracle.java @@ -49,8 +49,7 @@ private class TLPWhereReproducer extends AbstractComparisonReproducer errorReproducer(String originalQueryString, String firstQueryString, + String secondQueryString, String thirdQueryString, boolean orderBy, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> { + ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, globalState); + if (firstQueryString != null) { + ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, + new ArrayList<>(), !orderBy, globalState, errors); + } + }; + StringBuilder sb = new StringBuilder(); + renderQueryLines(sb, originalQueryString, firstQueryString, secondQueryString, thirdQueryString, orderBy); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + public TLPWhereOracle(G state, TLPWhereGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { throw new IllegalArgumentException("Null variables used to initialize test oracle."); @@ -140,7 +158,7 @@ public void check() throws SQLException { try { firstResultSet = ComparatorHelper.getResultSetFirstColumnAsString(originalQueryString, errors, state); } catch (AssertionError unexpectedError) { - reproducer = new TLPWhereReproducer(null, null, null, originalQueryString, false, + reproducer = errorReproducer(originalQueryString, null, null, null, false, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } @@ -165,13 +183,13 @@ public void check() throws SQLException { secondResultSet = ComparatorHelper.getCombinedResultSet(firstQueryString, secondQueryString, thirdQueryString, combinedString, !orderBy, state, errors); } catch (AssertionError unexpectedError) { - reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, - originalQueryString, orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + reproducer = errorReproducer(originalQueryString, firstQueryString, secondQueryString, thirdQueryString, + orderBy, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); throw unexpectedError; } reproducer = new TLPWhereReproducer(firstQueryString, secondQueryString, thirdQueryString, originalQueryString, - orderBy, null); + orderBy); ComparatorHelper.assumeResultSetsAreEqual(firstResultSet, secondResultSet, originalQueryString, combinedString, state); } diff --git a/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java b/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java new file mode 100644 index 000000000..e9390943a --- /dev/null +++ b/src/sqlancer/common/oracle/UnexpectedErrorReproducer.java @@ -0,0 +1,79 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; + +import sqlancer.Reproducer; +import sqlancer.SQLGlobalState; + +/** + * Reproducer for a bug that is an unexpected DBMS error, rather than a violation of oracle logic. When a statement run + * by the oracle raises an error the oracle did not expect, the bug is that error. Reduction re-runs the statement + * execution against the reduced database and reports whether the same error still fires. The oracle supplies how to + * re-run its execution as an {@link Execution} functional interface. + * + *

+ * For an oracle to use this reproducer on its unexpected errors, they must surface as {@link AssertionError}s, even + * though they likely originated as {@link SQLException}s. This is because + * {@link UnexpectedErrorReproducer#bugStillTriggers} treats a {@link SQLException} as a replay failure that means "bug + * does not trigger" (e.g. dropped connection, removed table). + * + * @param + * the DBMS-specific global state class + */ +public final class UnexpectedErrorReproducer> implements Reproducer { + + private final Execution execution; + private final String expectedErrorMessage; + private final String queryLines; + + @FunctionalInterface + public interface Execution { + /** + * Re-runs the oracle's execution against the reduced database. The bug is treated as still present if the run + * continues to raise an {@link AssertionError} with the same message. + * + * @param globalState + * the state whose connection points at the reduced database + * + * @throws SQLException + * if a DBMS interaction fails for a reason other than the recorded bug (e.g. a connection or setup + * failure during replay), which counts as the bug no longer triggering + */ + void execute(G globalState) throws SQLException; + } + + /** + * @param execution + * re-runs the oracle's execution against the reduced database + * @param expectedErrorMessage + * the message of the error the original bug was. The bug is treated as still present if the + * {@link Execution} continues to raise an {@link AssertionError} with the same message. + * @param queryLines + * the failing queries as commented lines (each ending in a line separator), for the reduced test case + */ + public UnexpectedErrorReproducer(Execution execution, String expectedErrorMessage, String queryLines) { + this.execution = execution; + this.expectedErrorMessage = expectedErrorMessage; + this.queryLines = queryLines; + } + + @Override + public boolean bugStillTriggers(G globalState) { + try { + execution.execute(globalState); + } catch (AssertionError unexpectedError) { + // the same error reproduces the bug; a different one is an artifact of the reduction (e.g. a removed table) + return expectedErrorMessage.equals(TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + } catch (SQLException | RuntimeException e) { + return false; + } + // the error no longer fires + return false; + } + + @Override + public String getBugInformation() { + return "-- On the database set up by the statements above, the following queries trigger an unexpected error" + + " with message: " + expectedErrorMessage + System.lineSeparator() + queryLines; + } +} From 96ab07d6fd1abde8a6f53bfa59b0f6736fbff79b Mon Sep 17 00:00:00 2001 From: splf Date: Fri, 31 Jul 2026 10:07:42 +0500 Subject: [PATCH 841/856] Expect the to_char error for RN combined with other formats PostgreSQL rejects a numeric format string that combines RN with any other pattern, for example to_char(1.5, 'RN0'), with the error "RN" is incompatible with other formats. The expression generator can produce such format strings, so this error reached the caller as an unexpected one and aborted the test run. --- src/sqlancer/postgres/gen/PostgresCommon.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sqlancer/postgres/gen/PostgresCommon.java b/src/sqlancer/postgres/gen/PostgresCommon.java index eeb160a56..180b4e449 100644 --- a/src/sqlancer/postgres/gen/PostgresCommon.java +++ b/src/sqlancer/postgres/gen/PostgresCommon.java @@ -123,6 +123,7 @@ private static List getToCharFunctionErrors() { errors.add("cannot use \"PR\" and \"S\"/\"PL\"/\"MI\"/\"SG\" together"); errors.add("is not a number"); errors.add("\"EEEE\" must be the last pattern used"); + errors.add("is incompatible with other formats"); return errors; } From adc0875fe76c4d48939fc4be2a0097ee715e521a Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Tue, 28 Jul 2026 16:40:37 +0800 Subject: [PATCH 842/856] Add support for DECIMAL(M, D) columns in MySQL EET --- src/sqlancer/mysql/MySQLSchema.java | 12 ++- .../mysql/ast/MySQLCastOperation.java | 80 +++++++++++++++++-- .../mysql/gen/MySQLTableGenerator.java | 21 +++-- .../mysql/oracle/MySQLEETTransformer.java | 27 ++++--- .../mysql/MySQLToStringVisitorTest.java | 6 +- .../mysql/ast/MySQLCaseOperatorTest.java | 4 +- 6 files changed, 120 insertions(+), 30 deletions(-) diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index c8a30614f..ab0ba4543 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -57,6 +57,7 @@ public static class MySQLColumn extends AbstractTableColumn getTableColumns(SQLConnection con, String table String columnName = rs.getString("COLUMN_NAME"); String dataType = rs.getString("DATA_TYPE"); int precision = rs.getInt("NUMERIC_PRECISION"); + int scale = rs.getInt("NUMERIC_SCALE"); boolean isPrimaryKey = rs.getString("COLUMN_KEY").equals("PRI"); - MySQLColumn c = new MySQLColumn(columnName, getColumnType(dataType), isPrimaryKey, precision); + MySQLColumn c = new MySQLColumn(columnName, getColumnType(dataType), isPrimaryKey, precision, + scale); columns.add(c); } } diff --git a/src/sqlancer/mysql/ast/MySQLCastOperation.java b/src/sqlancer/mysql/ast/MySQLCastOperation.java index b71d0498c..07c45a429 100644 --- a/src/sqlancer/mysql/ast/MySQLCastOperation.java +++ b/src/sqlancer/mysql/ast/MySQLCastOperation.java @@ -1,21 +1,91 @@ package sqlancer.mysql.ast; +import java.util.Objects; + public class MySQLCastOperation implements MySQLExpression { private final MySQLExpression expr; private final CastType type; - public enum CastType { - SIGNED, UNSIGNED, - // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never - // evaluated, so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). - CHAR, FLOAT, DOUBLE, DECIMAL; + /** + * A MySQL {@code CAST} target type. The non-{@code DECIMAL} kinds are interned singletons; {@code DECIMAL} may + * additionally carry an {@code (M, D)} precision/scale (via {@link #decimal}) so that a + * {@code CAST(... AS DECIMAL(M, D))} can reproduce a column's exact type. This is relied on by the EET oracle's + * type-pinning casts (see {@code MySQLEETTransformer}). + */ + public static final class CastType { + + // CHAR, FLOAT, DOUBLE and DECIMAL are used only by the EET oracle's type-pinning casts and are never evaluated, + // so MySQLConstant.castAs does not support them; they must not be returned by getRandom(). + public static final CastType SIGNED = new CastType(Kind.SIGNED); + public static final CastType UNSIGNED = new CastType(Kind.UNSIGNED); + public static final CastType CHAR = new CastType(Kind.CHAR); + public static final CastType FLOAT = new CastType(Kind.FLOAT); + public static final CastType DOUBLE = new CastType(Kind.DOUBLE); + public static final CastType DECIMAL = new CastType(Kind.DECIMAL); + + private enum Kind { + SIGNED, UNSIGNED, CHAR, FLOAT, DOUBLE, DECIMAL + } + + private final Kind kind; + private final Integer precision; // DECIMAL only, otherwise null + private final Integer scale; // DECIMAL only, otherwise null + + private CastType(Kind kind) { + this(kind, null, null); + } + + private CastType(Kind kind, Integer precision, Integer scale) { + this.kind = kind; + this.precision = precision; + this.scale = scale; + } + + // A DECIMAL(precision, scale) cast target. + public static CastType decimal(int precision, int scale) { + return new CastType(Kind.DECIMAL, precision, scale); + } public static CastType getRandom() { return SIGNED; // return Randomly.fromOptions(CastType.SIGNED, CastType.UNSIGNED); } + public Integer getPrecision() { + return precision; + } + + public Integer getScale() { + return scale; + } + + @Override + public String toString() { + if (precision == null) { + return kind.name(); + } + return kind.name() + "(" + precision + ", " + scale + ")"; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof CastType)) { + return false; + } + CastType other = (CastType) obj; + return kind == other.kind && Objects.equals(precision, other.precision) + && Objects.equals(scale, other.scale); + } + + @Override + public int hashCode() { + return Objects.hash(kind, precision, scale); + } + } public MySQLCastOperation(MySQLExpression expr, CastType type) { diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 40e325041..b7c9a3566 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -347,11 +347,11 @@ private void appendType(MySQLDataType randomType) { break; case FLOAT: sb.append("FLOAT"); - optionallyAddPrecisionAndScale(sb); + optionallyAddFloatingPointPrecisionAndScale(sb); break; case DOUBLE: sb.append(Randomly.fromOptions("DOUBLE", "FLOAT")); - optionallyAddPrecisionAndScale(sb); + optionallyAddFloatingPointPrecisionAndScale(sb); break; default: throw new AssertionError(); @@ -368,13 +368,20 @@ private void appendType(MySQLDataType randomType) { } } - private void optionallyAddPrecisionAndScale(StringBuilder sb) { - // The EET oracle's type inference assumes FLOAT/DOUBLE/DECIMAL columns are created without (M, D) (see - // MySQLEETTransformer#inferColumnType), so precision/scale is omitted while EET is active. This restriction can - // be lifted once (M, D) is tracked through the codebase and reflected in the CAST target types. + // FLOAT(M, D)/DOUBLE(M, D) is deprecated and cannot be reproduced as a CAST target, so the EET oracle's type + // inference relies on FLOAT/DOUBLE columns being created without (M, D) (see MySQLEETTransformer#inferColumnType); + // it is therefore omitted while EET is active. DECIMAL(M, D) has no such restriction: the EET oracle tracks its + // (M, D) and reproduces it via CAST(... AS DECIMAL(M, D)), so it keeps using optionallyAddPrecisionAndScale. + private void optionallyAddFloatingPointPrecisionAndScale(StringBuilder sb) { boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.EET); - if (Randomly.getBoolean() && !MySQLBugs.bug99183 && !eetActive) { + if (!eetActive) { + optionallyAddPrecisionAndScale(sb); + } + } + + private void optionallyAddPrecisionAndScale(StringBuilder sb) { + if (Randomly.getBoolean() && !MySQLBugs.bug99183) { sb.append("("); // The maximum number of digits (M) for DECIMAL is 65 long m = Randomly.getNotCachedInteger(1, 65); diff --git a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java index 2e4394eec..22c0302df 100644 --- a/src/sqlancer/mysql/oracle/MySQLEETTransformer.java +++ b/src/sqlancer/mysql/oracle/MySQLEETTransformer.java @@ -5,6 +5,7 @@ import java.util.stream.Collectors; import sqlancer.common.oracle.EETTransformer; +import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; @@ -33,11 +34,11 @@ * *

* MySQL's expression generator is untyped, so type inference/generation works with a subset of MySQL's CAST target - * types ({@link CastType}): {@link #inferType} conservatively classifies AST nodes into that domain (returning - * {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type of a random expression by wrapping - * it in a CAST. + * types ({@link CastType}, which carries {@code (M, D)} for DECIMAL): {@link #inferType} conservatively classifies AST + * nodes into that domain (returning {@code null} when uncertain), and {@link #generateExpressionOfType} pins the type + * of a random expression by wrapping it in a CAST. */ -public class MySQLEETTransformer extends EETTransformer { +public class MySQLEETTransformer extends EETTransformer { private static final boolean BOOLEAN = true; private static final boolean SCALAR = false; @@ -212,20 +213,22 @@ private CastType inferConstantType(MySQLConstant constant) { } private CastType inferColumnType(MySQLColumnReference ref) { - switch (ref.getColumn().getType()) { + MySQLColumn column = ref.getColumn(); + switch (column.getType()) { case INT: return CastType.SIGNED; // the table generator never creates UNSIGNED INT columns case VARCHAR: return CastType.CHAR; - // FLOAT/DOUBLE/DECIMAL columns are created without (M, D) while EET is active, so the plain - // CAST target below matches the column's type. Reintroducing (M, D) for better coverage would - // require tracking it here and emitting the exact precision/scale in the CAST. case FLOAT: + // FLOAT/DOUBLE columns are created without (M, D) while EET is active (the (M, D) form is deprecated and + // not a valid CAST target), so the plain CAST target matches the column's type. return CastType.FLOAT; case DOUBLE: return CastType.DOUBLE; case DECIMAL: - return CastType.DECIMAL; + // DECIMAL columns may carry (M, D); CAST(... AS DECIMAL(M, D)) reproduces the column's exact type. The + // schema reports (M, D) even for a plain DECIMAL column (defaulting to (10, 0)). + return CastType.decimal(column.getPrecision(), column.getScale()); default: return null; } @@ -259,7 +262,8 @@ private CastType inferCaseType(MySQLCaseOperator caseOp) { /** * The common type of several result-type-determining subexpressions, or {@code null} if they do not have the same - * inferrable type (a conservative under-approximation of MySQL's aggregation rules). + * inferrable type (a conservative under-approximation of MySQL's aggregation rules). Two DECIMAL subexpressions + * with differing {@code (M, D)} therefore yield {@code null} rather than a guessed aggregate. * * @param exprs * the result-type-determining subexpressions @@ -270,7 +274,8 @@ private CastType commonType(MySQLExpression... exprs) { CastType common = null; for (MySQLExpression expr : exprs) { CastType type = inferType(expr); - if (type == null || common != null && type != common) { + // equals (not ==) so two DECIMAL types with matching (M, D) but distinct instances compare as equal. + if (type == null || common != null && !type.equals(common)) { return null; } common = type; diff --git a/test/sqlancer/mysql/MySQLToStringVisitorTest.java b/test/sqlancer/mysql/MySQLToStringVisitorTest.java index 3f991e3f1..d39c2695c 100644 --- a/test/sqlancer/mysql/MySQLToStringVisitorTest.java +++ b/test/sqlancer/mysql/MySQLToStringVisitorTest.java @@ -17,7 +17,7 @@ public class MySQLToStringVisitorTest { @Test void visitAggregateToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCount = new MySQLAggregate(List.of(aRef), MySQLAggregate.MySQLAggregateFunction.COUNT); @@ -35,7 +35,7 @@ void visitAggregateToString() { @Test void visitAggregateWithDistinctToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference aRef = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); MySQLAggregate aggrCountDistinct = new MySQLAggregate(List.of(aRef), @@ -57,7 +57,7 @@ void visitAggregateWithDistinctToString() { @Test void visitCaseWhenToString() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLConstant.createNullConstant()); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); diff --git a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java index 674ab027e..757241445 100644 --- a/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java +++ b/test/sqlancer/mysql/ast/MySQLCaseOperatorTest.java @@ -14,7 +14,7 @@ public class MySQLCaseOperatorTest { @Test void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createIntConstant(1)); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); @@ -29,7 +29,7 @@ void getExpectedValue_switchConditionMatchesWhen_ReturnsThen() { @Test void getExpectedValue_switchConditionHasNoMatches_ReturnsElse() { - MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0); + MySQLSchema.MySQLColumn aCol = new MySQLSchema.MySQLColumn("a", MySQLSchema.MySQLDataType.INT, false, 0, 0); MySQLColumnReference switchExpr = new MySQLColumnReference(aCol, MySQLIntConstant.createNullConstant()); List whenExprs = List.of(MySQLIntConstant.createIntConstant(1), MySQLIntConstant.createIntConstant(2)); From 9b739c0501f848c0a0907728356bbe43bed926c4 Mon Sep 17 00:00:00 2001 From: splf Date: Mon, 3 Aug 2026 12:49:45 +0500 Subject: [PATCH 843/856] Do not return null when PostgreSQL tablespaces are disabled --- .../gen/PostgresTableSpaceGenerator.java | 7 ++++-- .../gen/TestPostgresTableSpaceGenerator.java | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java diff --git a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java index 3890d5160..1be99ca61 100644 --- a/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java +++ b/src/sqlancer/postgres/gen/PostgresTableSpaceGenerator.java @@ -1,5 +1,6 @@ package sqlancer.postgres.gen; +import sqlancer.IgnoreMeException; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; import sqlancer.postgres.PostgresGlobalState; @@ -19,10 +20,12 @@ public PostgresTableSpaceGenerator(PostgresGlobalState globalState) { } public static SQLQueryAdapter generate(PostgresGlobalState globalState) { - // Skip tablespace generation if the option is disabled + // PostgresProvider.mapActions does not schedule this action when the option is disabled, but QPG + // selects actions by index without consulting the schedule, so the generator has to report that + // it has nothing to generate. PostgresOptions options = globalState.getDbmsSpecificOptions(); if (!options.isTestTablespaces()) { - return null; + throw new IgnoreMeException(); } return new PostgresTableSpaceGenerator(globalState).generateTableSpace(); } diff --git a/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java b/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java new file mode 100644 index 000000000..c2f626041 --- /dev/null +++ b/test/sqlancer/postgres/gen/TestPostgresTableSpaceGenerator.java @@ -0,0 +1,25 @@ +package sqlancer.postgres.gen; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +import sqlancer.IgnoreMeException; +import sqlancer.postgres.PostgresGlobalState; +import sqlancer.postgres.PostgresOptions; + +class TestPostgresTableSpaceGenerator { + + @Test + void generateIsSkippedWhenTablespacesAreDisabled() { + PostgresGlobalState state = new PostgresGlobalState(); + state.setDbmsSpecificOptions(new PostgresOptions() { + @Override + public boolean isTestTablespaces() { + return false; + } + }); + + assertThrows(IgnoreMeException.class, () -> PostgresTableSpaceGenerator.generate(state)); + } +} From 6ac84559193fa5f004ac6c18a020ade4a86ba68b Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 13:37:48 +0800 Subject: [PATCH 844/856] Add support for DELETE statements in EET (with MySQL-specific implementation), including generic DML infrastructure for future extensions to other DML statements --- src/sqlancer/common/gen/EETDMLGenerator.java | 155 +++++++++++++++++ src/sqlancer/common/oracle/EETDMLOracle.java | 158 ++++++++++++++++++ src/sqlancer/mysql/MySQLErrors.java | 17 ++ src/sqlancer/mysql/MySQLGlobalState.java | 9 + src/sqlancer/mysql/MySQLOracleFactory.java | 10 ++ .../mysql/gen/MySQLExpressionGenerator.java | 22 ++- .../mysql/gen/MySQLTableGenerator.java | 14 +- 7 files changed, 378 insertions(+), 7 deletions(-) create mode 100644 src/sqlancer/common/gen/EETDMLGenerator.java create mode 100644 src/sqlancer/common/oracle/EETDMLOracle.java diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java new file mode 100644 index 000000000..293574a36 --- /dev/null +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -0,0 +1,155 @@ +package sqlancer.common.gen; + +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.oracle.EETTransformer; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * Generator interface used by {@link sqlancer.common.oracle.EETDMLOracle}, the DML counterpart of {@link EETGenerator}. + * It supplies methods which generate the transformable expressions, create the DBMS-specific {@link EETTransformer} + * that rewrites them, and produce the SQL of the statements the oracle uses to observe the database state a statement + * produces (an approach drawn from the DQE oracle). + * + *

+ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which + * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run + * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are + * produced. + * + *

+ * Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ +public interface EETDMLGenerator, T extends AbstractTable, C extends AbstractTableColumn> { + + /** Name of the auxiliary column that uniquely identifies each row. */ + String ROW_ID_COLUMN = "rowid"; + + /** + * Restricts this generator to the given tables (a single table, for the DML statement under test) and their + * columns. + * + * @param tables + * the tables (and, implicitly, columns) the generated statement operates on + * + * @return this generator + */ + EETDMLGenerator setTablesAndColumns(AbstractTables tables); + + /** + * Generates a fresh random boolean expression over the current tables' columns, used as the DML statement's WHERE + * predicate. + * + * @return a fresh random boolean expression + */ + E generateBooleanExpression(); + + /** + * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's + * expressions into semantically equivalent ones. + * + * @return a DBMS-specific {@link EETTransformer} + */ + EETTransformer createTransformer(); + + // --- DBMS-specific primitives --- + + /** + * Renders an expression to its DBMS-specific SQL string. + * + * @param expr + * the expression to render + * + * @return the SQL text of {@code expr} + */ + String asString(E expr); + + /** + * SQL that assigns every existing row of {@code table} a distinct, stable identifier in the {@link #ROW_ID_COLUMN} + * column. DBMS-specific because it names the DBMS's UUID-generating function. + * + * @param table + * the table whose rows are stamped + * + * @return the SQL statement + */ + String stampRowIdsStatement(T table); + + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- + + /** + * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}. + * + * @param table + * the table to add the column to + * + * @return the SQL statement + */ + default String addRowIdColumnStatement(T table) { + return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " VARCHAR(36)"; + } + + /** + * SQL that drops the auxiliary {@link #ROW_ID_COLUMN} column from {@code table}. + * + * @param table + * the table to drop the column from + * + * @return the SQL statement + */ + default String dropRowIdColumnStatement(T table) { + return "ALTER TABLE " + table.getName() + " DROP COLUMN " + ROW_ID_COLUMN; + } + + /** + * SQL that selects the {@link #ROW_ID_COLUMN} of every row of {@code table} (the surviving-row snapshot). + * + * @param table + * the table to snapshot + * + * @return the SQL statement; its first result column must be the identifiers + */ + default String selectRowIdsStatement(T table) { + return "SELECT " + ROW_ID_COLUMN + " FROM " + table.getName(); + } + + /** + * SQL that deletes the rows of {@code table} matching {@code predicate}. + * + * @param table + * the table to delete from + * @param predicate + * the WHERE predicate; rendered via {@link #asString} + * + * @return the SQL statement + */ + default String deleteStatement(T table, E predicate) { + return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); + } + + /** + * SQL that starts a transaction, so a statement's effect can be observed and then undone. + * + * @return the SQL statement + */ + default String beginTransactionStatement() { + return "BEGIN"; + } + + /** + * SQL that rolls the current transaction back, undoing the statement's effect. + * + * @return the SQL statement + */ + default String rollbackTransactionStatement() { + return "ROLLBACK"; + } +} diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java new file mode 100644 index 000000000..33f9ff411 --- /dev/null +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -0,0 +1,158 @@ +package sqlancer.common.oracle; + +import java.sql.SQLException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import sqlancer.ComparatorHelper; +import sqlancer.IgnoreMeException; +import sqlancer.Randomly; +import sqlancer.SQLGlobalState; +import sqlancer.common.ast.newast.Expression; +import sqlancer.common.gen.EETDMLGenerator; +import sqlancer.common.query.ExpectedErrors; +import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.schema.AbstractSchema; +import sqlancer.common.schema.AbstractTable; +import sqlancer.common.schema.AbstractTableColumn; +import sqlancer.common.schema.AbstractTables; + +/** + * EET (Equivalent Expression Transformation) oracle for DML statements, based on "Detecting Logic Bugs in Database + * Engines via Equivalent Expression Transformation" (Jiang & Su, OSDI'24). + * + *

+ * Whereas {@link EETOracle} transforms a SELECT and compares the two result sets, this oracle transforms a DML + * statement and compares the two database states produced. + * + *

+ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which + * uniquely identifies each row, and each statement is executed inside a transaction that is rolled back, so the two + * statements can be compared against the same starting state without permanently modifying the database. For a DELETE, + * the state is captured as the set of surviving row identifiers. Because rolling back a statement requires a + * transactional storage engine, the DBMS-specific setup must ensure only such engines are used while this oracle is + * active. + * + *

+ * Only DELETE is currently supported. Statement reduction is not yet implemented (there is no + * {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database reduction. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific schema class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + * @param + * the DBMS-specific global state class + */ +public class EETDMLOracle, S extends AbstractSchema, T extends AbstractTable, C extends AbstractTableColumn, G extends SQLGlobalState> + implements TestOracle { + + private final G state; + private EETDMLGenerator gen; + private final EETTransformer transformer; + private final ExpectedErrors errors; + + private String generatedQueryString; + + public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { + if (state == null || gen == null || expectedErrors == null) { + throw new IllegalArgumentException("Null variables used to initialize test oracle."); + } + this.state = state; + this.gen = gen; + this.transformer = gen.createTransformer(); + this.errors = expectedErrors; + } + + @Override + public void check() throws SQLException { + List tables = state.getSchema().getDatabaseTables(); + if (tables.isEmpty()) { + throw new IgnoreMeException(); + } + // DELETE targets a single table, so operate on exactly one; confining the generator to it keeps the predicate + // from referencing another table's columns (which would render invalid single-table DML). + T table = Randomly.fromList(tables); + gen = gen.setTablesAndColumns(new AbstractTables<>(List.of(table))); + + E predicate = gen.generateBooleanExpression(); + // The WHERE predicate is evaluated in a boolean context. + E transformedPredicate = transformer.transform(predicate, true); + + String originalDelete = gen.deleteStatement(table, predicate); + String transformedDelete = gen.deleteStatement(table, transformedPredicate); + generatedQueryString = originalDelete; + + // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: + // the ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak + // the column into the next iteration and cause cascading duplicate-column failures. + new SQLQueryAdapter(gen.addRowIdColumnStatement(table), true).execute(state); + try { + // Stamp identifiers once, in autocommit mode, before both DELETEs run: both then observe the same rows. + new SQLQueryAdapter(gen.stampRowIdsStatement(table)).execute(state); + + Set originalSurvivors = executeDeleteAndSnapshot(table, originalDelete); + Set transformedSurvivors = executeDeleteAndSnapshot(table, transformedDelete); + + if (!originalSurvivors.equals(transformedSurvivors)) { + throw new AssertionError( + mismatchMessage(originalDelete, transformedDelete, originalSurvivors, transformedSurvivors)); + } + } finally { + new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), true).execute(state); + } + } + + /** + * Executes {@code deleteStatement} inside a transaction that is always rolled back, and returns the set of row + * identifiers surviving the DELETE (the resulting database state). A DBMS error expected by the oracle aborts the + * whole check ({@link IgnoreMeException}) rather than being reported, matching {@link EETOracle}'s handling; an + * unexpected error surfaces as a bug ({@link AssertionError}, thrown by the query adapter). + * + * @param table + * the table being deleted from + * @param deleteStatement + * the DELETE statement to execute + * + * @return the set of row identifiers surviving the DELETE + * + * @throws SQLException + * if a DBMS interaction fails + */ + private Set executeDeleteAndSnapshot(T table, String deleteStatement) throws SQLException { + new SQLQueryAdapter(gen.beginTransactionStatement()).execute(state); + try { + // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. + boolean succeeded = new SQLQueryAdapter(deleteStatement, errors).execute(state); + if (!succeeded) { + // The DELETE hit an error the oracle tolerates; do not compare states (as EETOracle does for SELECT). + throw new IgnoreMeException(); + } + return new HashSet<>( + ComparatorHelper.getResultSetFirstColumnAsString(gen.selectRowIdsStatement(table), errors, state)); + } finally { + new SQLQueryAdapter(gen.rollbackTransactionStatement()).execute(state); + } + } + + private static String mismatchMessage(String originalDelete, String transformedDelete, + Set originalSurvivors, Set transformedSurvivors) { + return new StringBuilder() + .append("-- The original and transformed DELETE statements left the database in different states") + .append(" (different sets of surviving rows):").append(System.lineSeparator()).append("-- original (") + .append(originalSurvivors.size()).append(" rows survive): ").append(originalDelete).append(';') + .append(System.lineSeparator()).append("-- transformed (").append(transformedSurvivors.size()) + .append(" rows survive): ").append(transformedDelete).append(';').append(System.lineSeparator()) + .toString(); + } + + @Override + public String getLastQueryString() { + return generatedQueryString; + } +} diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index cb9ad4f01..44316dc3f 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -65,4 +65,21 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { errors.addAll(getInsertUpdateErrors()); } + public static List getDMLErrors() { + ArrayList errors = new ArrayList<>(); + + // WHERE-clause type coercion (e.g. string -> number) is only a warning in SELECT but a hard error in + // DELETE/UPDATE under strict sql_mode (MySQL 1292). A semantics-preserving transform may benignly change + // whether it fires, so it is tolerated rather than flagged. + errors.add("Truncated incorrect"); + // Foreign key constraint failure when deleting/updating a referenced row. + errors.add("a foreign key constraint fails"); + + return errors; + } + + public static void addDMLErrors(ExpectedErrors errors) { + errors.addAll(getDMLErrors()); + } + } diff --git a/src/sqlancer/mysql/MySQLGlobalState.java b/src/sqlancer/mysql/MySQLGlobalState.java index 10132b57c..a34861cbd 100644 --- a/src/sqlancer/mysql/MySQLGlobalState.java +++ b/src/sqlancer/mysql/MySQLGlobalState.java @@ -16,4 +16,13 @@ public boolean usesPQS() { return getDbmsSpecificOptions().oracles.stream().anyMatch(o -> o == MySQLOracleFactory.PQS); } + public boolean usesEET() { + return getDbmsSpecificOptions().getTestOracleFactory().stream() + .anyMatch(o -> o == MySQLOracleFactory.EET || o == MySQLOracleFactory.EET_DML); + } + + public boolean usesEETDML() { + return getDbmsSpecificOptions().getTestOracleFactory().stream().anyMatch(o -> o == MySQLOracleFactory.EET_DML); + } + } diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index ed5ddc489..d3d4b20b1 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -5,6 +5,7 @@ import sqlancer.OracleFactory; import sqlancer.common.oracle.CERTOracle; +import sqlancer.common.oracle.EETDMLOracle; import sqlancer.common.oracle.EETOracle; import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; @@ -92,5 +93,14 @@ public TestOracle create(MySQLGlobalState globalState) throws .withRegex(MySQLErrors.getExpressionRegexErrors()).build(); return new EETOracle<>(globalState, gen, expectedErrors); } + }, + EET_DML { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); + ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) + .withRegex(MySQLErrors.getExpressionRegexErrors()).with(MySQLErrors.getDMLErrors()).build(); + return new EETDMLOracle<>(globalState, gen, expectedErrors); + } }; } diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index da304ac67..5a694bd78 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -9,6 +9,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.gen.CERTGenerator; +import sqlancer.common.gen.EETDMLGenerator; import sqlancer.common.gen.EETGenerator; import sqlancer.common.gen.TLPWhereGenerator; import sqlancer.common.gen.UntypedExpressionGenerator; @@ -19,6 +20,7 @@ import sqlancer.mysql.MySQLSchema.MySQLColumn; import sqlancer.mysql.MySQLSchema.MySQLRowValue; import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLVisitor; import sqlancer.mysql.ast.MySQLAggregate; import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; import sqlancer.mysql.ast.MySQLBetweenOperation; @@ -52,7 +54,8 @@ public class MySQLExpressionGenerator extends UntypedExpressionGenerator implements TLPWhereGenerator, CERTGenerator, - EETGenerator { + EETGenerator, + EETDMLGenerator { private final MySQLGlobalState state; private MySQLRowValue rowVal; @@ -364,10 +367,25 @@ boolean mutateOr(MySQLSelect select) { } } - // --- EET oracle --- + // --- EET oracle (including DML) --- @Override public EETTransformer createTransformer() { return new MySQLEETTransformer(this); } + + // --- EET DML only --- + + @Override + public String asString(MySQLExpression expr) { + return MySQLVisitor.asString(expr); + } + + @Override + public String stampRowIdsStatement(MySQLTable table) { + // MySQL's UUID() gives each existing row a distinct value in a single statement. Stamping happens once, before + // both rolled-back DELETE runs, so both observe identical identifiers; the standard-SQL statements (add/drop + // column, delete, snapshot, transaction control) use EETDMLGenerator's defaults. + return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); + } } diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index b7c9a3566..0746056ad 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -198,7 +198,10 @@ private void appendTableOptions() { // "NDB": java.sql.SQLSyntaxErrorException: Unknown storage engine 'NDB' // "EXAMPLE": java.sql.SQLSyntaxErrorException: Unknown storage engine 'EXAMPLE' // "MERGE": java.sql.SQLException: Table 't0' is read only - String fromOptions = Randomly.fromOptions("InnoDB", "MyISAM", "MEMORY", "HEAP", "CSV", "ARCHIVE"); + // The EET DML oracle rolls back each statement to compare database states, which requires a + // transactional engine, so only InnoDB is used while it is active. + String fromOptions = globalState.usesEETDML() ? "InnoDB" + : Randomly.fromOptions("InnoDB", "MyISAM", "MEMORY", "HEAP", "CSV", "ARCHIVE"); this.engine = MySQLEngine.get(fromOptions); sb.append("ENGINE = "); sb.append(fromOptions); @@ -362,7 +365,8 @@ private void appendType(MySQLDataType randomType) { } if (Randomly.getBoolean() && !globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() .anyMatch(o -> o == MySQLOracleFactory.TLP_WHERE || o == MySQLOracleFactory.PQS - || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET)) { + || o == MySQLOracleFactory.DQP || o == MySQLOracleFactory.EET + || o == MySQLOracleFactory.EET_DML)) { sb.append(" ZEROFILL"); } } @@ -373,9 +377,9 @@ private void appendType(MySQLDataType randomType) { // it is therefore omitted while EET is active. DECIMAL(M, D) has no such restriction: the EET oracle tracks its // (M, D) and reproduces it via CAST(... AS DECIMAL(M, D)), so it keeps using optionallyAddPrecisionAndScale. private void optionallyAddFloatingPointPrecisionAndScale(StringBuilder sb) { - boolean eetActive = globalState.getDbmsSpecificOptions().getTestOracleFactory().stream() - .anyMatch(o -> o == MySQLOracleFactory.EET); - if (!eetActive) { + // Both EET oracles rely on the same type inference (MySQLEETTransformer), so both omit FLOAT(M, D)/DOUBLE(M, + // D). + if (!globalState.usesEET()) { optionallyAddPrecisionAndScale(sb); } } From 514f14959e295f56981514248bf8a96660596ccd Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 15:10:55 +0800 Subject: [PATCH 845/856] Add support for LIMIT on DELETE statements in EET --- src/sqlancer/common/gen/EETDMLGenerator.java | 32 ++++++++++++++++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 13 ++++++-- 2 files changed, 40 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 293574a36..375877a63 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -1,5 +1,8 @@ package sqlancer.common.gen; +import java.util.ArrayList; +import java.util.List; + import sqlancer.common.ast.newast.Expression; import sqlancer.common.oracle.EETTransformer; import sqlancer.common.schema.AbstractTable; @@ -122,17 +125,40 @@ default String selectRowIdsStatement(T table) { } /** - * SQL that deletes the rows of {@code table} matching {@code predicate}. + * SQL that deletes the rows of {@code table} matching {@code predicate}, optionally limited to the first + * {@code limit} rows. + * + *

+ * When {@code limit} is non-null, the statement is ordered by {@code orderByColumns} followed by + * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even when + * the ordering columns tie), so the "first {@code limit}" rows are identical for the original and + * transformed statements. Varying the ordering columns exercises more access + * paths than the row id alone would. The caller must pass the same {@code orderByColumns} and {@code limit} + * to both statements; neither is transformed. * * @param table * the table to delete from * @param predicate * the WHERE predicate; rendered via {@link #asString} + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty); only used when {@code limit} is + * non-null + * @param limit + * the maximum number of rows to delete, or {@code null} for no limit * * @return the SQL statement */ - default String deleteStatement(T table, E predicate) { - return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); + default String deleteStatement(T table, E predicate, List orderByColumns, Integer limit) { + String statement = "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); + if (limit != null) { + List orderBy = new ArrayList<>(); + for (C column : orderByColumns) { + orderBy.add(column.getName()); + } + orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above + statement += " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; + } + return statement; } /** diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 33f9ff411..792bc52c0 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -84,8 +84,17 @@ public void check() throws SQLException { // The WHERE predicate is evaluated in a boolean context. E transformedPredicate = transformer.transform(predicate, true); - String originalDelete = gen.deleteStatement(table, predicate); - String transformedDelete = gen.deleteStatement(table, transformedPredicate); + // Optionally cap the DELETE with a LIMIT. The limit and its ordering (a random column subset, made a total + // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped + // row set is deterministic and equal across the runs while still exercising varied orderings. + Integer limit = null; + List orderByColumns = List.of(); + if (Randomly.getBoolean()) { + limit = (int) Randomly.getNotCachedInteger(0, 10); + orderByColumns = Randomly.subset(table.getColumns()); + } + String originalDelete = gen.deleteStatement(table, predicate, orderByColumns, limit); + String transformedDelete = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); generatedQueryString = originalDelete; // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: From 0dea938244b227cc03c900982c7730a78544f002 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 15:33:50 +0800 Subject: [PATCH 846/856] Add existing insert/update errors to EET DML oracle expecteed errors --- src/sqlancer/common/gen/EETDMLGenerator.java | 9 ++++----- src/sqlancer/common/oracle/EETDMLOracle.java | 14 ++++++++++---- src/sqlancer/mysql/MySQLErrors.java | 2 +- src/sqlancer/mysql/MySQLOracleFactory.java | 6 +++++- 4 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 375877a63..82888e070 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -130,11 +130,10 @@ default String selectRowIdsStatement(T table) { * *

* When {@code limit} is non-null, the statement is ordered by {@code orderByColumns} followed by - * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even when - * the ordering columns tie), so the "first {@code limit}" rows are identical for the original and - * transformed statements. Varying the ordering columns exercises more access - * paths than the row id alone would. The caller must pass the same {@code orderByColumns} and {@code limit} - * to both statements; neither is transformed. + * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even + * when the ordering columns tie), so the "first {@code limit}" rows are identical for the original and transformed + * statements. Varying the ordering columns exercises more access paths than the row id alone would. The caller must + * pass the same {@code orderByColumns} and {@code limit} to both statements; neither is transformed. * * @param table * the table to delete from diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 792bc52c0..ce41039d6 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -99,11 +99,17 @@ public void check() throws SQLException { // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: // the ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak - // the column into the next iteration and cause cascading duplicate-column failures. - new SQLQueryAdapter(gen.addRowIdColumnStatement(table), true).execute(state); + // the column into the next iteration and cause cascading duplicate-column failures. The row-identity setup + // touches rows, so — like the DELETE itself — it can raise tolerated errors (e.g. functional-index maintenance + // truncation); such an error aborts the iteration (IgnoreMeException) rather than being reported as a bug. + if (!new SQLQueryAdapter(gen.addRowIdColumnStatement(table), errors, true).execute(state)) { + throw new IgnoreMeException(); + } try { // Stamp identifiers once, in autocommit mode, before both DELETEs run: both then observe the same rows. - new SQLQueryAdapter(gen.stampRowIdsStatement(table)).execute(state); + if (!new SQLQueryAdapter(gen.stampRowIdsStatement(table), errors).execute(state)) { + throw new IgnoreMeException(); + } Set originalSurvivors = executeDeleteAndSnapshot(table, originalDelete); Set transformedSurvivors = executeDeleteAndSnapshot(table, transformedDelete); @@ -113,7 +119,7 @@ public void check() throws SQLException { mismatchMessage(originalDelete, transformedDelete, originalSurvivors, transformedSurvivors)); } } finally { - new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), true).execute(state); + new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); } } diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 44316dc3f..0b6eb8284 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -66,7 +66,7 @@ public static void addInsertUpdateErrors(ExpectedErrors errors) { } public static List getDMLErrors() { - ArrayList errors = new ArrayList<>(); + ArrayList errors = new ArrayList<>(getInsertUpdateErrors()); // WHERE-clause type coercion (e.g. string -> number) is only a warning in SELECT but a hard error in // DELETE/UPDATE under strict sql_mode (MySQL 1292). A semantics-preserving transform may benignly change diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index d3d4b20b1..8b37efb13 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -99,7 +99,11 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { MySQLExpressionGenerator gen = new MySQLExpressionGenerator(globalState); ExpectedErrors expectedErrors = ExpectedErrors.newErrors().with(MySQLErrors.getExpressionErrors()) - .withRegex(MySQLErrors.getExpressionRegexErrors()).with(MySQLErrors.getDMLErrors()).build(); + .withRegex(MySQLErrors.getExpressionRegexErrors()) + // The DML statements and the row-identity setup (adding/stamping the auxiliary column) touch rows, + // so they can raise the full range of DML errors — e.g. functional-index maintenance truncation — + // beyond the SELECT-based expression errors. + .with(MySQLErrors.getDMLErrors()).build(); return new EETDMLOracle<>(globalState, gen, expectedErrors); } }; From 85d837a7215911cd11418e8e1caec6ca93c633f7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Sat, 8 Aug 2026 10:39:52 +0800 Subject: [PATCH 847/856] Increase generality of EETDMLGenerator.addRowIdColumnStatement --- src/sqlancer/common/gen/EETDMLGenerator.java | 15 ++++++++++++--- .../mysql/gen/MySQLExpressionGenerator.java | 6 ++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 82888e070..b56580a9d 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -77,7 +77,7 @@ public interface EETDMLGenerator, T extends AbstractTabl /** * SQL that assigns every existing row of {@code table} a distinct, stable identifier in the {@link #ROW_ID_COLUMN} - * column. DBMS-specific because it names the DBMS's UUID-generating function. + * column. For example, a 36-character UUID string. * * @param table * the table whose rows are stamped @@ -86,10 +86,19 @@ public interface EETDMLGenerator, T extends AbstractTabl */ String stampRowIdsStatement(T table); + /** + * The SQL type of the auxiliary {@link #ROW_ID_COLUMN} column. It must be able to hold the identifiers that + * {@link #stampRowIdsStatement} produces, so it belongs with that statement as the other half of the row-id + * representation. For example, {@code VARCHAR(36)} would fit a 36-character UUID string. + * + * @return the column type + */ + String rowIdColumnType(); + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- /** - * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}. + * SQL that adds the auxiliary {@link #ROW_ID_COLUMN} column to {@code table}, typed as {@link #rowIdColumnType}. * * @param table * the table to add the column to @@ -97,7 +106,7 @@ public interface EETDMLGenerator, T extends AbstractTabl * @return the SQL statement */ default String addRowIdColumnStatement(T table) { - return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " VARCHAR(36)"; + return "ALTER TABLE " + table.getName() + " ADD COLUMN " + ROW_ID_COLUMN + " " + rowIdColumnType(); } /** diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 5a694bd78..9a62c15ba 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -388,4 +388,10 @@ public String stampRowIdsStatement(MySQLTable table) { // column, delete, snapshot, transaction control) use EETDMLGenerator's defaults. return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); } + + @Override + public String rowIdColumnType() { + // Holds a 36-character UUID string produced by stampRowIdsStatement. + return "VARCHAR(36)"; + } } From 7b9b0549aa945ef583f94fed0b90bef7dad318b7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 19:48:07 +0800 Subject: [PATCH 848/856] Implement UPDATE support for EET DML oracle --- src/sqlancer/common/gen/EETDMLGenerator.java | 111 ++++++++--- src/sqlancer/common/oracle/EETDMLOracle.java | 174 +++++++++++++----- .../mysql/gen/MySQLExpressionGenerator.java | 18 +- 3 files changed, 231 insertions(+), 72 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index b56580a9d..9fb21ea35 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Map; import sqlancer.common.ast.newast.Expression; import sqlancer.common.oracle.EETTransformer; @@ -19,7 +20,8 @@ * Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are - * produced. + * produced. The resulting state is compared as a full post-image (each surviving row's identifier and content column + * values), which covers every DML statement: a DELETE removes rows from it, an UPDATE changes values in it. * *

* Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. @@ -55,6 +57,15 @@ public interface EETDMLGenerator, T extends AbstractTabl */ E generateBooleanExpression(); + /** + * Generates a fresh set of {@code column = value} assignments over the current tables' columns, used as an UPDATE + * statement's SET clause. The columns are a random non-empty subset and each value is a fresh random expression; + * both the columns and their assigned expressions are transformed by the oracle. + * + * @return the assignments, as {@code (column, value expression)} pairs (at least one) + */ + List> generateSetAssignments(); + /** * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's * expressions into semantically equivalent ones. @@ -122,27 +133,32 @@ default String dropRowIdColumnStatement(T table) { } /** - * SQL that selects the {@link #ROW_ID_COLUMN} of every row of {@code table} (the surviving-row snapshot). + * SQL that reads back the full post-image of {@code table}: the {@link #ROW_ID_COLUMN} identifier and every content + * column of every surviving row, ordered by the (unique) identifier so the two statements' snapshots align + * row-for-row. + * + *

+ * This single value-level snapshot is the comparison surface for all DML statements: a DELETE removes rows from it, + * an UPDATE changes column values in it. Row identity alone (which the identifier already captures) would suffice + * for DELETE, but not for UPDATE, where the two runs could touch the same rows yet write different values. * * @param table * the table to snapshot * - * @return the SQL statement; its first result column must be the identifiers + * @return the SQL statement; its first result column is the identifier, followed by {@code table}'s content columns */ - default String selectRowIdsStatement(T table) { - return "SELECT " + ROW_ID_COLUMN + " FROM " + table.getName(); + default String selectPostImageStatement(T table) { + List selected = new ArrayList<>(); + selected.add(ROW_ID_COLUMN); + for (C column : table.getColumns()) { + selected.add(column.getName()); + } + return "SELECT " + String.join(", ", selected) + " FROM " + table.getName() + " ORDER BY " + ROW_ID_COLUMN; } /** * SQL that deletes the rows of {@code table} matching {@code predicate}, optionally limited to the first - * {@code limit} rows. - * - *

- * When {@code limit} is non-null, the statement is ordered by {@code orderByColumns} followed by - * {@link #ROW_ID_COLUMN} as a tiebreaker. Because the identifiers are unique, this is always a total order (even - * when the ordering columns tie), so the "first {@code limit}" rows are identical for the original and transformed - * statements. Varying the ordering columns exercises more access paths than the row id alone would. The caller must - * pass the same {@code orderByColumns} and {@code limit} to both statements; neither is transformed. + * {@code limit} rows (see {@link #orderByLimitClause}). * * @param table * the table to delete from @@ -157,16 +173,67 @@ default String selectRowIdsStatement(T table) { * @return the SQL statement */ default String deleteStatement(T table, E predicate, List orderByColumns, Integer limit) { - String statement = "DELETE FROM " + table.getName() + " WHERE " + asString(predicate); - if (limit != null) { - List orderBy = new ArrayList<>(); - for (C column : orderByColumns) { - orderBy.add(column.getName()); - } - orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above - statement += " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; + return "DELETE FROM " + table.getName() + " WHERE " + asString(predicate) + + orderByLimitClause(orderByColumns, limit); + } + + /** + * SQL that updates the rows of {@code table} matching {@code predicate}, setting each column in {@code assignments} + * to its assigned value expression, optionally limited to the first {@code limit} rows (see + * {@link #orderByLimitClause}). + * + * @param table + * the table to update + * @param assignments + * the {@code (column, value expression)} pairs to assign; each value is rendered via {@link #asString} + * @param predicate + * the WHERE predicate; rendered via {@link #asString} + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty); only used when {@code limit} is + * non-null + * @param limit + * the maximum number of rows to update, or {@code null} for no limit + * + * @return the SQL statement + */ + default String updateStatement(T table, List> assignments, E predicate, List orderByColumns, + Integer limit) { + List setClauses = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + setClauses.add(assignment.getKey().getName() + " = " + asString(assignment.getValue())); + } + return "UPDATE " + table.getName() + " SET " + String.join(", ", setClauses) + " WHERE " + asString(predicate) + + orderByLimitClause(orderByColumns, limit); + } + + /** + * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and + * {@link #updateStatement}, or the empty string when {@code limit} is null. + * + *

+ * The rows are ordered by {@code orderByColumns} followed by {@link #ROW_ID_COLUMN} as a tiebreaker. Because the + * identifiers are unique, this is always a total order (even when the ordering columns tie), so the "first + * {@code limit}" rows are identical for the original and transformed statements. Varying the ordering columns + * exercises more access paths than the row id alone would. The caller must pass the same {@code orderByColumns} and + * {@code limit} to both statements; neither is transformed. + * + * @param orderByColumns + * the columns to order by before the row-id tiebreaker (may be empty) + * @param limit + * the maximum number of rows, or {@code null} for no limit (yielding an empty clause) + * + * @return the {@code ORDER BY ... LIMIT n} clause, or the empty string when {@code limit} is null + */ + default String orderByLimitClause(List orderByColumns, Integer limit) { + if (limit == null) { + return ""; + } + List orderBy = new ArrayList<>(); + for (C column : orderByColumns) { + orderBy.add(column.getName()); } - return statement; + orderBy.add(ROW_ID_COLUMN); // unique tiebreaker: guarantees a total order regardless of the columns above + return " ORDER BY " + String.join(", ", orderBy) + " LIMIT " + limit; } /** diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index ce41039d6..3443c9756 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -1,11 +1,11 @@ package sqlancer.common.oracle; import java.sql.SQLException; -import java.util.HashSet; +import java.util.AbstractMap; +import java.util.ArrayList; import java.util.List; -import java.util.Set; +import java.util.Map; -import sqlancer.ComparatorHelper; import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.SQLGlobalState; @@ -13,6 +13,7 @@ import sqlancer.common.gen.EETDMLGenerator; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLQueryAdapter; +import sqlancer.common.query.SQLancerResultSet; import sqlancer.common.schema.AbstractSchema; import sqlancer.common.schema.AbstractTable; import sqlancer.common.schema.AbstractTableColumn; @@ -29,14 +30,17 @@ *

* Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row, and each statement is executed inside a transaction that is rolled back, so the two - * statements can be compared against the same starting state without permanently modifying the database. For a DELETE, - * the state is captured as the set of surviving row identifiers. Because rolling back a statement requires a - * transactional storage engine, the DBMS-specific setup must ensure only such engines are used while this oracle is - * active. + * statements can be compared against the same starting state without permanently modifying the database. The state is + * captured as a full post-image: each surviving row's identifier together with its content column values, ordered by + * the identifier. This single value-level surface covers every DML statement — a DELETE removes rows from it, an UPDATE + * changes values in it (row identity alone would suffice for DELETE, but not for UPDATE, which also transforms the + * written values). Because rolling back a statement requires a transactional storage engine, the DBMS-specific setup + * must ensure only such engines are used while this oracle is active. * *

- * Only DELETE is currently supported. Statement reduction is not yet implemented (there is no - * {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database reduction. + * DELETE and UPDATE are currently supported (one is chosen at random per check). Statement reduction is not yet + * implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database + * reduction. * * @param * the DBMS-specific expression class @@ -75,8 +79,9 @@ public void check() throws SQLException { if (tables.isEmpty()) { throw new IgnoreMeException(); } - // DELETE targets a single table, so operate on exactly one; confining the generator to it keeps the predicate - // from referencing another table's columns (which would render invalid single-table DML). + // A DML statement targets a single table, so operate on exactly one; confining the generator to it keeps the + // predicate and value expressions from referencing another table's columns (which would render invalid + // single-table DML). T table = Randomly.fromList(tables); gen = gen.setTablesAndColumns(new AbstractTables<>(List.of(table))); @@ -84,7 +89,7 @@ public void check() throws SQLException { // The WHERE predicate is evaluated in a boolean context. E transformedPredicate = transformer.transform(predicate, true); - // Optionally cap the DELETE with a LIMIT. The limit and its ordering (a random column subset, made a total + // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. Integer limit = null; @@ -93,30 +98,47 @@ public void check() throws SQLException { limit = (int) Randomly.getNotCachedInteger(0, 10); orderByColumns = Randomly.subset(table.getColumns()); } - String originalDelete = gen.deleteStatement(table, predicate, orderByColumns, limit); - String transformedDelete = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); - generatedQueryString = originalDelete; - - // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: - // the ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak - // the column into the next iteration and cause cascading duplicate-column failures. The row-identity setup - // touches rows, so — like the DELETE itself — it can raise tolerated errors (e.g. functional-index maintenance - // truncation); such an error aborts the iteration (IgnoreMeException) rather than being reported as a bug. + + String originalStatement; + String transformedStatement; + if (Randomly.getBoolean()) { + // UPDATE also transforms the written values: each SET value expression is transformed in a scalar context. + List> assignments = gen.generateSetAssignments(); + List> transformedAssignments = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + E transformedValue = transformer.transform(assignment.getValue(), false); + transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); + } + originalStatement = gen.updateStatement(table, assignments, predicate, orderByColumns, limit); + transformedStatement = gen.updateStatement(table, transformedAssignments, transformedPredicate, + orderByColumns, limit); + } else { + originalStatement = gen.deleteStatement(table, predicate, orderByColumns, limit); + transformedStatement = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); + } + generatedQueryString = originalStatement; + + // The post-image select reads the identifier plus every content column of the table, in that order. + int columnCount = table.getColumns().size() + 1; + + // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the + // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the + // column and cause cascading duplicate-column failures if (!new SQLQueryAdapter(gen.addRowIdColumnStatement(table), errors, true).execute(state)) { throw new IgnoreMeException(); } try { - // Stamp identifiers once, in autocommit mode, before both DELETEs run: both then observe the same rows. + // Stamp identifiers once, in autocommit mode, before both runs: both then observe the same rows. if (!new SQLQueryAdapter(gen.stampRowIdsStatement(table), errors).execute(state)) { throw new IgnoreMeException(); } - Set originalSurvivors = executeDeleteAndSnapshot(table, originalDelete); - Set transformedSurvivors = executeDeleteAndSnapshot(table, transformedDelete); + List> originalImage = executeAndSnapshotPostImage(table, originalStatement, columnCount); + List> transformedImage = executeAndSnapshotPostImage(table, transformedStatement, columnCount); - if (!originalSurvivors.equals(transformedSurvivors)) { + if (!originalImage.equals(transformedImage)) { throw new AssertionError( - mismatchMessage(originalDelete, transformedDelete, originalSurvivors, transformedSurvivors)); + mismatchMessage(originalStatement, transformedStatement, originalImage, transformedImage)); } } finally { new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); @@ -124,46 +146,102 @@ public void check() throws SQLException { } /** - * Executes {@code deleteStatement} inside a transaction that is always rolled back, and returns the set of row - * identifiers surviving the DELETE (the resulting database state). A DBMS error expected by the oracle aborts the - * whole check ({@link IgnoreMeException}) rather than being reported, matching {@link EETOracle}'s handling; an - * unexpected error surfaces as a bug ({@link AssertionError}, thrown by the query adapter). + * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: + * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A + * DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug or unexpected error + * surfaces as {@link AssertionError}. * * @param table - * the table being deleted from - * @param deleteStatement - * the DELETE statement to execute + * the table being modified + * @param statement + * the DML statement to execute + * @param columnCount + * the number of columns the post-image select returns (identifier plus content columns) * - * @return the set of row identifiers surviving the DELETE + * @return the post-image, as one string list (identifier followed by content column values) per surviving row * * @throws SQLException - * if a DBMS interaction fails + * if a DBMS interaction other than running {@code statement} fails; an error from {@code statement} + * itself instead surfaces as {@link IgnoreMeException} or {@link AssertionError} */ - private Set executeDeleteAndSnapshot(T table, String deleteStatement) throws SQLException { + private List> executeAndSnapshotPostImage(T table, String statement, int columnCount) + throws SQLException { new SQLQueryAdapter(gen.beginTransactionStatement()).execute(state); try { // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. - boolean succeeded = new SQLQueryAdapter(deleteStatement, errors).execute(state); + boolean succeeded = new SQLQueryAdapter(statement, errors).execute(state); if (!succeeded) { - // The DELETE hit an error the oracle tolerates; do not compare states (as EETOracle does for SELECT). + // The statement hit an error the oracle tolerates; do not compare states (as EETOracle does for + // SELECT). throw new IgnoreMeException(); } - return new HashSet<>( - ComparatorHelper.getResultSetFirstColumnAsString(gen.selectRowIdsStatement(table), errors, state)); + return snapshotPostImage(gen.selectPostImageStatement(table), columnCount); } finally { new SQLQueryAdapter(gen.rollbackTransactionStatement()).execute(state); } } - private static String mismatchMessage(String originalDelete, String transformedDelete, - Set originalSurvivors, Set transformedSurvivors) { + /** + * Reads the post-image produced by {@code selectStatement} into one string list per row (each column via + * {@code getString}). A DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug + * or unexpected error surfaces as {@link AssertionError}. + * + * @param selectStatement + * the post-image select to read; its columns are the identifier followed by the content columns + * @param columnCount + * the number of columns to read from each row + * + * @return the read rows, in the select's order + * + * @throws SQLException + * if cleanup fails (errors thrown elsewhere will always be rethrown as {@link IgnoreMeException} or + * {@link AssertionError}) + */ + private List> snapshotPostImage(String selectStatement, int columnCount) throws SQLException { + List> rows = new ArrayList<>(); + SQLQueryAdapter q = new SQLQueryAdapter(selectStatement, errors, true, + state.getOptions().canonicalizeSqlString()); + SQLancerResultSet result = null; + try { + result = q.executeAndGet(state); + if (result == null) { + throw new IgnoreMeException(); + } + while (result.next()) { + List row = new ArrayList<>(columnCount); + for (int i = 1; i <= columnCount; i++) { + row.add(result.getString(i)); + } + rows.add(row); + } + } catch (Exception e) { + if (e instanceof IgnoreMeException) { + throw e; + } + Throwable current = e; + while (current != null) { + if (current.getMessage() != null && errors.errorIsExpected(current.getMessage())) { + throw new IgnoreMeException(); + } + current = current.getCause(); + } + throw new AssertionError(selectStatement, e); + } finally { + if (result != null && !result.isClosed()) { + result.close(); + } + } + return rows; + } + + private static String mismatchMessage(String originalStatement, String transformedStatement, + List> originalImage, List> transformedImage) { return new StringBuilder() - .append("-- The original and transformed DELETE statements left the database in different states") - .append(" (different sets of surviving rows):").append(System.lineSeparator()).append("-- original (") - .append(originalSurvivors.size()).append(" rows survive): ").append(originalDelete).append(';') - .append(System.lineSeparator()).append("-- transformed (").append(transformedSurvivors.size()) - .append(" rows survive): ").append(transformedDelete).append(';').append(System.lineSeparator()) - .toString(); + .append("-- The original and transformed statements left the database in different states") + .append(" (different post-images):").append(System.lineSeparator()).append("-- original (") + .append(originalImage.size()).append(" rows): ").append(originalStatement).append(';') + .append(System.lineSeparator()).append("-- transformed (").append(transformedImage.size()) + .append(" rows): ").append(transformedStatement).append(';').append(System.lineSeparator()).toString(); } @Override diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9a62c15ba..9fbaef12a 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -1,7 +1,9 @@ package sqlancer.mysql.gen; +import java.util.AbstractMap; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -254,6 +256,18 @@ public MySQLExpression generateBooleanExpression() { return generateExpression(); } + @Override + public List> generateSetAssignments() { + List> assignments = new ArrayList<>(); + for (MySQLColumn column : Randomly.nonEmptySubset(columns)) { + // As with the normal UPDATE workload, the value is an arbitrary expression (not type-matched to the + // column); + // any resulting type/range error is on the oracle's expected-error allow-list. + assignments.add(new AbstractMap.SimpleEntry<>(column, generateExpression())); + } + return assignments; + } + @Override public MySQLSelect generateSelect() { return new MySQLSelect(); @@ -384,8 +398,8 @@ public String asString(MySQLExpression expr) { @Override public String stampRowIdsStatement(MySQLTable table) { // MySQL's UUID() gives each existing row a distinct value in a single statement. Stamping happens once, before - // both rolled-back DELETE runs, so both observe identical identifiers; the standard-SQL statements (add/drop - // column, delete, snapshot, transaction control) use EETDMLGenerator's defaults. + // both rolled-back statement runs, so both observe identical identifiers; the standard-SQL statements (add/drop + // column, delete/update, snapshot, transaction control) use EETDMLGenerator's defaults. return String.format("UPDATE %s SET %s = UUID()", table.getName(), ROW_ID_COLUMN); } From 1e53207c5770081e746e67a9ee1e5dac6abc5d94 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 29 Jul 2026 20:11:53 +0800 Subject: [PATCH 849/856] Add row discrepancy information to test case logs for EET DML oracle --- src/sqlancer/common/oracle/EETDMLOracle.java | 65 +++++++++++++++++--- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 3443c9756..fbb2d2d3b 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -3,8 +3,12 @@ import java.sql.SQLException; import java.util.AbstractMap; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -61,6 +65,7 @@ public class EETDMLOracle, S extends AbstractSchema transformer; private final ExpectedErrors errors; + private static final int MAX_DIFF_ROWS_REPORTED = 10; // max differing post-image rows displayed in report log private String generatedQueryString; public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { @@ -137,8 +142,8 @@ public void check() throws SQLException { List> transformedImage = executeAndSnapshotPostImage(table, transformedStatement, columnCount); if (!originalImage.equals(transformedImage)) { - throw new AssertionError( - mismatchMessage(originalStatement, transformedStatement, originalImage, transformedImage)); + throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, originalImage, + transformedImage)); } } finally { new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); @@ -234,14 +239,56 @@ private List> snapshotPostImage(String selectStatement, int columnC return rows; } - private static String mismatchMessage(String originalStatement, String transformedStatement, + private String mismatchMessage(T table, String originalStatement, String transformedStatement, List> originalImage, List> transformedImage) { - return new StringBuilder() - .append("-- The original and transformed statements left the database in different states") - .append(" (different post-images):").append(System.lineSeparator()).append("-- original (") - .append(originalImage.size()).append(" rows): ").append(originalStatement).append(';') - .append(System.lineSeparator()).append("-- transformed (").append(transformedImage.size()) - .append(" rows): ").append(transformedStatement).append(';').append(System.lineSeparator()).toString(); + List header = new ArrayList<>(); + header.add(EETDMLGenerator.ROW_ID_COLUMN); + for (C column : table.getColumns()) { + header.add(column.getName()); + } + + Map> originalByRowId = indexByRowId(originalImage); + Map> transformedByRowId = indexByRowId(transformedImage); + Set allRowIds = new TreeSet<>(); + allRowIds.addAll(originalByRowId.keySet()); + allRowIds.addAll(transformedByRowId.keySet()); + + String nl = System.lineSeparator(); + StringBuilder message = new StringBuilder() + .append("-- The original and transformed statements left the database in different states.").append(nl) + .append("-- original: ").append(originalStatement).append(';').append(nl).append("-- transformed: ") + .append(transformedStatement).append(';').append(nl).append("-- differing post-image rows (") + .append(String.join(", ", header)).append("):").append(nl); + int shown = 0; + for (String rowId : allRowIds) { + List originalRow = originalByRowId.get(rowId); + List transformedRow = transformedByRowId.get(rowId); + if (Objects.equals(originalRow, transformedRow)) { + continue; + } + if (shown == MAX_DIFF_ROWS_REPORTED) { + message.append("-- ... (further differences omitted)").append(nl); + break; + } + message.append("-- original: ").append(renderRow(originalRow)).append(nl); + message.append("-- transformed: ").append(renderRow(transformedRow)).append(nl); + shown++; + } + return message.toString(); + } + + // Indexes a post-image by its row identifier (the first column of each row) + private static Map> indexByRowId(List> image) { + Map> byRowId = new LinkedHashMap<>(); + for (List row : image) { + byRowId.put(row.get(0), row); + } + return byRowId; + } + + // Renders a post-image row for the finding message, or "(row absent)" when the row is missing on that side + private static String renderRow(List row) { + return row == null ? "(row absent)" : row.toString(); } @Override From 63f50b826a7b217ed12f4fdd8aeb10613ea166df Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 12 Aug 2026 09:28:31 +0800 Subject: [PATCH 850/856] Remove hard-coding of EET DML rowId column being at index 0; derive it instead --- src/sqlancer/common/gen/EETDMLGenerator.java | 26 ++++++++++++++++---- src/sqlancer/common/oracle/EETDMLOracle.java | 21 +++++++--------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 9fb21ea35..0c84901f3 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -145,15 +145,31 @@ default String dropRowIdColumnStatement(T table) { * @param table * the table to snapshot * - * @return the SQL statement; its first result column is the identifier, followed by {@code table}'s content columns + * @return the SQL statement; its result columns are those of {@link #postImageColumns}, in that order */ default String selectPostImageStatement(T table) { - List selected = new ArrayList<>(); - selected.add(ROW_ID_COLUMN); + return "SELECT " + String.join(", ", postImageColumns(table)) + " FROM " + table.getName() + " ORDER BY " + + ROW_ID_COLUMN; + } + + /** + * The columns a post-image row consists of, in the order {@link #selectPostImageStatement} returns them: the + * {@link #ROW_ID_COLUMN} identifier followed by {@code table}'s content columns. This is the sole definition of the + * post-image layout, so a consumer can find the identifier's position by looking up {@link #ROW_ID_COLUMN} here + * rather than assuming one. + * + * @param table + * the table being snapshot + * + * @return the post-image column names, in order + */ + default List postImageColumns(T table) { + List columns = new ArrayList<>(); + columns.add(ROW_ID_COLUMN); for (C column : table.getColumns()) { - selected.add(column.getName()); + columns.add(column.getName()); } - return "SELECT " + String.join(", ", selected) + " FROM " + table.getName() + " ORDER BY " + ROW_ID_COLUMN; + return columns; } /** diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index fbb2d2d3b..eb453d9b6 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -123,8 +123,7 @@ public void check() throws SQLException { } generatedQueryString = originalStatement; - // The post-image select reads the identifier plus every content column of the table, in that order. - int columnCount = table.getColumns().size() + 1; + int columnCount = gen.postImageColumns(table).size(); // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the @@ -241,14 +240,12 @@ private List> snapshotPostImage(String selectStatement, int columnC private String mismatchMessage(T table, String originalStatement, String transformedStatement, List> originalImage, List> transformedImage) { - List header = new ArrayList<>(); - header.add(EETDMLGenerator.ROW_ID_COLUMN); - for (C column : table.getColumns()) { - header.add(column.getName()); - } + List header = gen.postImageColumns(table); + // Where the identifier sits within a post-image row, per the layout the generator defines + int rowIdIndex = header.indexOf(EETDMLGenerator.ROW_ID_COLUMN); - Map> originalByRowId = indexByRowId(originalImage); - Map> transformedByRowId = indexByRowId(transformedImage); + Map> originalByRowId = indexByRowId(originalImage, rowIdIndex); + Map> transformedByRowId = indexByRowId(transformedImage, rowIdIndex); Set allRowIds = new TreeSet<>(); allRowIds.addAll(originalByRowId.keySet()); allRowIds.addAll(transformedByRowId.keySet()); @@ -277,11 +274,11 @@ private String mismatchMessage(T table, String originalStatement, String transfo return message.toString(); } - // Indexes a post-image by its row identifier (the first column of each row) - private static Map> indexByRowId(List> image) { + // Indexes a post-image by its row identifier, which each row holds at rowIdIndex + private static Map> indexByRowId(List> image, int rowIdIndex) { Map> byRowId = new LinkedHashMap<>(); for (List row : image) { - byRowId.put(row.get(0), row); + byRowId.put(row.get(rowIdIndex), row); } return byRowId; } From 1ce0341485a5052d9055a120cc3bf98321c3d0ee Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Wed, 12 Aug 2026 09:53:21 +0800 Subject: [PATCH 851/856] Refactor EET DML statement generation to use dedicated methods for update and delete --- src/sqlancer/common/oracle/EETDMLOracle.java | 86 ++++++++++++++++---- 1 file changed, 69 insertions(+), 17 deletions(-) diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index eb453d9b6..4731c04ed 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -104,23 +104,11 @@ public void check() throws SQLException { orderByColumns = Randomly.subset(table.getColumns()); } - String originalStatement; - String transformedStatement; - if (Randomly.getBoolean()) { - // UPDATE also transforms the written values: each SET value expression is transformed in a scalar context. - List> assignments = gen.generateSetAssignments(); - List> transformedAssignments = new ArrayList<>(); - for (Map.Entry assignment : assignments) { - E transformedValue = transformer.transform(assignment.getValue(), false); - transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); - } - originalStatement = gen.updateStatement(table, assignments, predicate, orderByColumns, limit); - transformedStatement = gen.updateStatement(table, transformedAssignments, transformedPredicate, - orderByColumns, limit); - } else { - originalStatement = gen.deleteStatement(table, predicate, orderByColumns, limit); - transformedStatement = gen.deleteStatement(table, transformedPredicate, orderByColumns, limit); - } + StatementPair statements = Randomly.getBoolean() + ? generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit) + : generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit); + String originalStatement = statements.original; + String transformedStatement = statements.transformed; generatedQueryString = originalStatement; int columnCount = gen.postImageColumns(table).size(); @@ -149,6 +137,70 @@ public void check() throws SQLException { } } + /** + * A DML statement and its transformed counterpart, which must leave the database in the same state. + */ + private static final class StatementPair { + private final String original; + private final String transformed; + + StatementPair(String original, String transformed) { + this.original = original; + this.transformed = transformed; + } + } + + /** + * Generates an UPDATE and its transformed counterpart. Besides the WHERE predicate, UPDATE also transforms the + * written values: each SET value expression is transformed in a scalar context. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the statement, empty if it is not capped by a limit + * @param limit + * the maximum number of rows to modify, or {@code null} for no limit + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateUpdateStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { + List> assignments = gen.generateSetAssignments(); + List> transformedAssignments = new ArrayList<>(); + for (Map.Entry assignment : assignments) { + E transformedValue = transformer.transform(assignment.getValue(), false); + transformedAssignments.add(new AbstractMap.SimpleEntry<>(assignment.getKey(), transformedValue)); + } + return new StatementPair(gen.updateStatement(table, assignments, predicate, orderByColumns, limit), + gen.updateStatement(table, transformedAssignments, transformedPredicate, orderByColumns, limit)); + } + + /** + * Generates a DELETE and its transformed counterpart, which differ only in their WHERE predicate. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the statement, empty if it is not capped by a limit + * @param limit + * the maximum number of rows to modify, or {@code null} for no limit + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateDeleteStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { + return new StatementPair(gen.deleteStatement(table, predicate, orderByColumns, limit), + gen.deleteStatement(table, transformedPredicate, orderByColumns, limit)); + } + /** * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A From 4cf31d15f2f841686365f0167ad489222eb4b90f Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 13 Aug 2026 13:40:00 +0800 Subject: [PATCH 852/856] Implement INSERT support for EET DML oracle --- src/sqlancer/common/gen/EETDMLGenerator.java | 72 +++++++++++++++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 61 ++++++++++++---- .../mysql/gen/MySQLExpressionGenerator.java | 16 +++++ 3 files changed, 130 insertions(+), 19 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 0c84901f3..c10c59b5a 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -19,9 +19,9 @@ *

* Adapted from the DQE oracle, state is observed with an auxiliary column ({@link EETDMLGenerator#ROW_ID_COLUMN}) which * uniquely identifies each row. The rows are stamped with identifiers once, before both executions of the statement run - * (each in a rolled-back transaction), so both executions observe the same identifiers regardless of how they are - * produced. The resulting state is compared as a full post-image (each surviving row's identifier and content column - * values), which covers every DML statement: a DELETE removes rows from it, an UPDATE changes values in it. + * (each in a rolled-back transaction), so both executions observe the same identifiers. The resulting state is compared + * as a full post-image (each surviving row's identifier and content column values), which covers any of the three DML + * statements (DELETE, UPDATE, INSERT). * *

* Most of these statements are standard SQL, likely common to most DBMSs, so are provided as {@code default} methods. @@ -66,6 +66,15 @@ public interface EETDMLGenerator, T extends AbstractTabl */ List> generateSetAssignments(); + /** + * Generates a fresh value expression for each content column of the current table, used as an INSERT statement's + * inserted values. The returned expressions are positionally aligned with {@link AbstractTable#getColumns()}, and + * each is transformed by the oracle. + * + * @return one fresh random value expression per content column, in {@link AbstractTable#getColumns()} order + */ + List generateInsertValues(); + /** * Creates a DBMS-specific {@link EETTransformer} backed by this generator, used to rewrite the statement's * expressions into semantically equivalent ones. @@ -106,6 +115,17 @@ public interface EETDMLGenerator, T extends AbstractTabl */ String rowIdColumnType(); + /** + * A SQL expression, evaluated once per source row of an {@code INSERT ... SELECT}, that derives the inserted row's + * {@link #ROW_ID_COLUMN} value from the source row's identifier. It must be deterministic (so both the original and + * transformed statements assign the same identifiers), unique per source row, and distinct from every existing + * identifier (so an inserted row never collides with the source row it was derived from in the post-image). DBMS- + * specific because it names a suitable derivation function (e.g. a hash of the source identifier). + * + * @return the SQL expression deriving an inserted row's identifier from the source row's {@link #ROW_ID_COLUMN} + */ + String insertedRowIdExpression(); + // --- Standard-SQL statements (override only where the DBMS's dialect differs) --- /** @@ -139,8 +159,9 @@ default String dropRowIdColumnStatement(T table) { * *

* This single value-level snapshot is the comparison surface for all DML statements: a DELETE removes rows from it, - * an UPDATE changes column values in it. Row identity alone (which the identifier already captures) would suffice - * for DELETE, but not for UPDATE, where the two runs could touch the same rows yet write different values. + * an UPDATE changes column values in it, an INSERT adds rows to it. Row identity alone (which the identifier + * already captures) would suffice for DELETE, but not for UPDATE, where the two runs could touch the same rows yet + * write different values. * * @param table * the table to snapshot @@ -222,6 +243,47 @@ default String updateStatement(T table, List> assignments, E pre + orderByLimitClause(orderByColumns, limit); } + /** + * SQL that inserts a new row into {@code table} for each source row (optionally filtered by {@code predicate}), + * setting each content column to its corresponding value in {@code values}. + * + *

+ * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value + * expressions reference the table's columns (the transformer injects column references into its equivalent + * sub-expressions), which are legal in a {@code SELECT} but not in a {@code VALUES} clause. Each inserted row's + * {@link #ROW_ID_COLUMN} is derived from its source row via {@link #insertedRowIdExpression()}, giving it a + * deterministic identifier that is unique and distinct from every existing one, so the two statements' post-images + * align (and inserted rows never collide with their source rows). + * + * @param table + * the table to insert into + * @param values + * one value expression per content column, positionally aligned with {@link AbstractTable#getColumns()}; + * each is rendered via {@link #asString} + * @param predicate + * the WHERE predicate filtering the source rows, or {@code null} to insert from every source row; + * rendered via {@link #asString} + * + * @return the SQL statement + */ + default String insertStatement(T table, List values, E predicate) { + List columnNames = new ArrayList<>(); + columnNames.add(ROW_ID_COLUMN); + List selectItems = new ArrayList<>(); + selectItems.add(insertedRowIdExpression()); + List columns = table.getColumns(); + for (int i = 0; i < columns.size(); i++) { + columnNames.add(columns.get(i).getName()); + selectItems.add(asString(values.get(i))); + } + String statement = "INSERT INTO " + table.getName() + " (" + String.join(", ", columnNames) + ") SELECT " + + String.join(", ", selectItems) + " FROM " + table.getName(); + if (predicate != null) { + statement += " WHERE " + asString(predicate); + } + return statement; + } + /** * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and * {@link #updateStatement}, or the empty string when {@code limit} is null. diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 4731c04ed..ba9b9d6c7 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -9,6 +9,7 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; +import java.util.function.Supplier; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -37,13 +38,15 @@ * statements can be compared against the same starting state without permanently modifying the database. The state is * captured as a full post-image: each surviving row's identifier together with its content column values, ordered by * the identifier. This single value-level surface covers every DML statement — a DELETE removes rows from it, an UPDATE - * changes values in it (row identity alone would suffice for DELETE, but not for UPDATE, which also transforms the - * written values). Because rolling back a statement requires a transactional storage engine, the DBMS-specific setup - * must ensure only such engines are used while this oracle is active. + * changes values in it, an INSERT adds rows to it (row identity alone would suffice for DELETE, but not for UPDATE, + * which also transforms the written values). Because rolling back a statement requires a transactional storage engine, + * the DBMS-specific setup must ensure only such engines are used while this oracle is active. * *

- * DELETE and UPDATE are currently supported (one is chosen at random per check). Statement reduction is not yet - * implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database + * DELETE, UPDATE and INSERT are currently supported (one is chosen at random per check). INSERT uses the + * {@code INSERT ... SELECT} form so its transformed value expressions may reference columns; each inserted row is given + * a deterministic identifier derived from its source row so the two runs' post-images align. Statement reduction is not + * yet implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database * reduction. * * @param @@ -97,16 +100,16 @@ public void check() throws SQLException { // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. - Integer limit = null; - List orderByColumns = List.of(); - if (Randomly.getBoolean()) { - limit = (int) Randomly.getNotCachedInteger(0, 10); - orderByColumns = Randomly.subset(table.getColumns()); - } + boolean withLimit = Randomly.getBoolean(); + Integer limit = withLimit ? (int) Randomly.getNotCachedInteger(0, 10) : null; + List orderByColumns = withLimit ? Randomly.subset(table.getColumns()) : List.of(); - StatementPair statements = Randomly.getBoolean() - ? generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit) - : generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit); + // Generators for the different kinds of statement this oracle supports. One is chosen at random per check + List> statementGenerators = List.of( + () -> generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit), + () -> generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit), + () -> generateInsertStatements(table, predicate, transformedPredicate)); + StatementPair statements = Randomly.fromList(statementGenerators).get(); String originalStatement = statements.original; String transformedStatement = statements.transformed; generatedQueryString = originalStatement; @@ -201,6 +204,36 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform gen.deleteStatement(table, transformedPredicate, orderByColumns, limit)); } + /** + * Generates an {@code INSERT ... SELECT} and its transformed counterpart. Besides the WHERE predicate, which + * filters the source rows and is optional here, INSERT also transforms each inserted value in a scalar context. + * + *

+ * Unlike DELETE and UPDATE, no limit is applied: {@link EETDMLGenerator#insertStatement} renders no ordering or + * limit, so one row is inserted per source row the predicate keeps. Nothing about INSERT rules a limit out — its + * source SELECT could carry the same ordering and limit the other statement kinds use, and the two runs would still + * read the same source rows — it is just not generated. + * + * @param table + * the table being modified + * @param predicate + * the WHERE predicate of the original statement + * @param transformedPredicate + * the transformed WHERE predicate, used by the transformed statement + * + * @return the original statement together with its transformed counterpart + */ + private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate) { + List values = gen.generateInsertValues(); + List transformedValues = new ArrayList<>(); + for (E value : values) { + transformedValues.add(transformer.transform(value, false)); + } + boolean withPredicate = Randomly.getBoolean(); + return new StatementPair(gen.insertStatement(table, values, withPredicate ? predicate : null), + gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null)); + } + /** * Executes {@code statement} inside a transaction that is always rolled back, and returns the resulting post-image: * the surviving rows' identifier and content column values, ordered by identifier (the resulting database state). A diff --git a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java index 9fbaef12a..baea11f65 100644 --- a/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLExpressionGenerator.java @@ -268,6 +268,14 @@ public List> generateSetAssignments() { return assignments; } + @Override + public List generateInsertValues() { + // One value per content column, in schema order (aligned with the INSERT column list). As with the normal + // INSERT workload, each value is an arbitrary expression (not type-matched to the column); any resulting + // type/range/constraint error is on the oracle's expected-error allow-list. + return columns.stream().map(c -> generateExpression()).collect(Collectors.toList()); + } + @Override public MySQLSelect generateSelect() { return new MySQLSelect(); @@ -408,4 +416,12 @@ public String rowIdColumnType() { // Holds a 36-character UUID string produced by stampRowIdsStatement. return "VARCHAR(36)"; } + + @Override + public String insertedRowIdExpression() { + // The source row's identifier with its dashes removed: deterministic (identical across both runs) and unique + // per + // source row. Fits the identifier column's VARCHAR(36). + return String.format("REPLACE(%s, '-', '')", ROW_ID_COLUMN); + } } From 7ead2e762106cdab3c71be9b094b1c42b033afb7 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 13 Aug 2026 13:50:17 +0800 Subject: [PATCH 853/856] Add support for LIMIT on source rows of INSERT ... SELECT statements in EET --- src/sqlancer/common/gen/EETDMLGenerator.java | 16 ++++-- src/sqlancer/common/oracle/EETDMLOracle.java | 53 ++++++++++++++------ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index c10c59b5a..557220d06 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -245,7 +245,8 @@ default String updateStatement(T table, List> assignments, E pre /** * SQL that inserts a new row into {@code table} for each source row (optionally filtered by {@code predicate}), - * setting each content column to its corresponding value in {@code values}. + * setting each content column to its corresponding value in {@code values}, optionally limited to the first + * {@code limit} source rows (see {@link #orderByLimitClause}). * *

* The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value @@ -263,10 +264,15 @@ default String updateStatement(T table, List> assignments, E pre * @param predicate * the WHERE predicate filtering the source rows, or {@code null} to insert from every source row; * rendered via {@link #asString} + * @param orderByColumns + * the columns to order the source rows by before the row-id tiebreaker (may be empty); only used when + * {@code limit} is non-null + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit * * @return the SQL statement */ - default String insertStatement(T table, List values, E predicate) { + default String insertStatement(T table, List values, E predicate, List orderByColumns, Integer limit) { List columnNames = new ArrayList<>(); columnNames.add(ROW_ID_COLUMN); List selectItems = new ArrayList<>(); @@ -281,12 +287,12 @@ default String insertStatement(T table, List values, E predicate) { if (predicate != null) { statement += " WHERE " + asString(predicate); } - return statement; + return statement + orderByLimitClause(orderByColumns, limit); } /** - * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement} and - * {@link #updateStatement}, or the empty string when {@code limit} is null. + * Renders the trailing {@code ORDER BY ... LIMIT n} clause shared by {@link #deleteStatement}, + * {@link #updateStatement} and {@link #insertStatement}, or the empty string when {@code limit} is null. * *

* The rows are ordered by {@code orderByColumns} followed by {@link #ROW_ID_COLUMN} as a tiebreaker. Because the diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index ba9b9d6c7..02bff8165 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -9,7 +9,6 @@ import java.util.Objects; import java.util.Set; import java.util.TreeSet; -import java.util.function.Supplier; import sqlancer.IgnoreMeException; import sqlancer.Randomly; @@ -100,16 +99,18 @@ public void check() throws SQLException { // Optionally cap the statement with a LIMIT. The limit and its ordering (a random column subset, made a total // order by the row-id tiebreaker) are decided once and applied identically to both statements, so the capped // row set is deterministic and equal across the runs while still exercising varied orderings. - boolean withLimit = Randomly.getBoolean(); - Integer limit = withLimit ? (int) Randomly.getNotCachedInteger(0, 10) : null; - List orderByColumns = withLimit ? Randomly.subset(table.getColumns()) : List.of(); + Integer limit = null; + List orderByColumns = List.of(); + if (Randomly.getBoolean()) { + limit = (int) Randomly.getNotCachedInteger(0, 10); + orderByColumns = Randomly.subset(table.getColumns()); + } // Generators for the different kinds of statement this oracle supports. One is chosen at random per check - List> statementGenerators = List.of( - () -> generateDeleteStatements(table, predicate, transformedPredicate, orderByColumns, limit), - () -> generateUpdateStatements(table, predicate, transformedPredicate, orderByColumns, limit), - () -> generateInsertStatements(table, predicate, transformedPredicate)); - StatementPair statements = Randomly.fromList(statementGenerators).get(); + List> statementGenerators = List.of(this::generateDeleteStatements, + this::generateUpdateStatements, this::generateInsertStatements); + StatementPair statements = Randomly.fromList(statementGenerators).generate(table, predicate, + transformedPredicate, orderByColumns, limit); String originalStatement = statements.original; String transformedStatement = statements.transformed; generatedQueryString = originalStatement; @@ -140,6 +141,22 @@ public void check() throws SQLException { } } + /** + * Generates a DML statement of one kind together with its transformed counterpart. The kinds share this signature + * so the oracle can pick one of them at random per check. + * + * @param + * the DBMS-specific expression class + * @param + * the DBMS-specific table class + * @param + * the DBMS-specific column class + */ + @FunctionalInterface + private interface DMLStatementGenerator { + StatementPair generate(T table, E predicate, E transformedPredicate, List orderByColumns, Integer limit); + } + /** * A DML statement and its transformed counterpart, which must leave the database in the same state. */ @@ -209,10 +226,7 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform * filters the source rows and is optional here, INSERT also transforms each inserted value in a scalar context. * *

- * Unlike DELETE and UPDATE, no limit is applied: {@link EETDMLGenerator#insertStatement} renders no ordering or - * limit, so one row is inserted per source row the predicate keeps. Nothing about INSERT rules a limit out — its - * source SELECT could carry the same ordering and limit the other statement kinds use, and the two runs would still - * read the same source rows — it is just not generated. + * The ordering and limit cap the source rows the statement reads, so it inserts one row per source row kept. * * @param table * the table being modified @@ -220,18 +234,25 @@ private StatementPair generateDeleteStatements(T table, E predicate, E transform * the WHERE predicate of the original statement * @param transformedPredicate * the transformed WHERE predicate, used by the transformed statement + * @param orderByColumns + * the columns ordering the source rows, empty if the statement is not capped by a limit + * @param limit + * the maximum number of source rows to insert from, or {@code null} for no limit * * @return the original statement together with its transformed counterpart */ - private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate) { + private StatementPair generateInsertStatements(T table, E predicate, E transformedPredicate, List orderByColumns, + Integer limit) { List values = gen.generateInsertValues(); List transformedValues = new ArrayList<>(); for (E value : values) { transformedValues.add(transformer.transform(value, false)); } boolean withPredicate = Randomly.getBoolean(); - return new StatementPair(gen.insertStatement(table, values, withPredicate ? predicate : null), - gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null)); + return new StatementPair( + gen.insertStatement(table, values, withPredicate ? predicate : null, orderByColumns, limit), + gen.insertStatement(table, transformedValues, withPredicate ? transformedPredicate : null, + orderByColumns, limit)); } /** From f46934ecb6ec699c0315631e38e71a8e97e8b27f Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Thu, 30 Jul 2026 10:11:10 +0800 Subject: [PATCH 854/856] Improve robustness of MySQL EET DML by ensuring InnoDB engine is always chosen --- src/sqlancer/mysql/gen/MySQLTableGenerator.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index 0746056ad..d27d6e681 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -164,7 +164,15 @@ public static List getRandomTableOptions() { } private void appendTableOptions() { - List tableOptions = TableOptions.getRandomTableOptions(); + List tableOptions = new ArrayList<>(TableOptions.getRandomTableOptions()); + // The EET DML oracle rolls back each statement to compare database states, which requires a transactional + // engine. The ENGINE option already forces InnoDB when the oracle is active (see the ENGINE case below), but it + // is only emitted when randomly chosen; otherwise the table would inherit the server's default engine, which is + // not guaranteed transactional. Force the option to always be present so the engine is never left to the + // server default. + if (globalState.usesEETDML() && !tableOptions.contains(TableOptions.ENGINE)) { + tableOptions.add(TableOptions.ENGINE); + } int i = 0; for (TableOptions o : tableOptions) { if (i++ != 0) { From f1e65caeb45e122bd968fdf7f7a6e783edd122d2 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 14 Aug 2026 10:27:43 +0800 Subject: [PATCH 855/856] Clarify EET DML insertStatement documentation --- src/sqlancer/common/gen/EETDMLGenerator.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/sqlancer/common/gen/EETDMLGenerator.java b/src/sqlancer/common/gen/EETDMLGenerator.java index 557220d06..9dbc9f0d5 100644 --- a/src/sqlancer/common/gen/EETDMLGenerator.java +++ b/src/sqlancer/common/gen/EETDMLGenerator.java @@ -249,12 +249,12 @@ default String updateStatement(T table, List> assignments, E pre * {@code limit} source rows (see {@link #orderByLimitClause}). * *

- * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because the transformed value - * expressions reference the table's columns (the transformer injects column references into its equivalent - * sub-expressions), which are legal in a {@code SELECT} but not in a {@code VALUES} clause. Each inserted row's - * {@link #ROW_ID_COLUMN} is derived from its source row via {@link #insertedRowIdExpression()}, giving it a - * deterministic identifier that is unique and distinct from every existing one, so the two statements' post-images - * align (and inserted rows never collide with their source rows). + * The {@code INSERT ... SELECT} form is used rather than {@code INSERT ... VALUES} because it reuses the source-row + * model already shared by {@link #deleteStatement} and {@link #updateStatement}, and because it offers two kinds of + * transformable expression in one statement (the inserted values and the WHERE predicate) rather than the values + * alone. Each inserted row's {@link #ROW_ID_COLUMN} is derived from its source row via + * {@link #insertedRowIdExpression()}, giving it a deterministic identifier that is unique and distinct from every + * existing one, so the two statements' post-images align (and inserted rows never collide with their source rows). * * @param table * the table to insert into From 3c707e66c0fef2ba22cd29aa724bab54b4872549 Mon Sep 17 00:00:00 2001 From: Thomas Morgan Date: Fri, 31 Jul 2026 11:44:33 +0800 Subject: [PATCH 856/856] Add reproducer to EET DML --- src/sqlancer/common/oracle/EETDMLOracle.java | 185 ++++++++++++++++--- 1 file changed, 158 insertions(+), 27 deletions(-) diff --git a/src/sqlancer/common/oracle/EETDMLOracle.java b/src/sqlancer/common/oracle/EETDMLOracle.java index 02bff8165..77a8b528a 100644 --- a/src/sqlancer/common/oracle/EETDMLOracle.java +++ b/src/sqlancer/common/oracle/EETDMLOracle.java @@ -12,6 +12,7 @@ import sqlancer.IgnoreMeException; import sqlancer.Randomly; +import sqlancer.Reproducer; import sqlancer.SQLGlobalState; import sqlancer.common.ast.newast.Expression; import sqlancer.common.gen.EETDMLGenerator; @@ -44,9 +45,9 @@ *

* DELETE, UPDATE and INSERT are currently supported (one is chosen at random per check). INSERT uses the * {@code INSERT ... SELECT} form so its transformed value expressions may reference columns; each inserted row is given - * a deterministic identifier derived from its source row so the two runs' post-images align. Statement reduction is not - * yet implemented (there is no {@link sqlancer.Reproducer Reproducer}), so the finding is reported without database - * reduction. + * a deterministic identifier derived from its source row so the two runs' post-images align. To support reduction, a + * {@link Reproducer} replays the whole comparison (adding and stamping the row-identifier column, running both + * statements in rolled-back transactions and comparing the post-images) against the reduced database. * * @param * the DBMS-specific expression class @@ -69,6 +70,93 @@ public class EETDMLOracle, S extends AbstractSchema reproducer; + + // The SQL and metadata to run and observe one DML comparison, captured as strings so a reproducer can replay it + // against a reduced database without the generator or live schema objects. + private static final class ComparisonQueries { + private final String originalStatement; + private final String transformedStatement; + private final String addRowIdColumn; + private final String stampRowIds; + private final String beginTransaction; + private final String rollback; + private final String dropRowIdColumn; + private final String selectPostImage; + private final int columnCount; + + ComparisonQueries(String originalStatement, String transformedStatement, String addRowIdColumn, + String stampRowIds, String beginTransaction, String rollback, String dropRowIdColumn, + String selectPostImage, int columnCount) { + this.originalStatement = originalStatement; + this.transformedStatement = transformedStatement; + this.addRowIdColumn = addRowIdColumn; + this.stampRowIds = stampRowIds; + this.beginTransaction = beginTransaction; + this.rollback = rollback; + this.dropRowIdColumn = dropRowIdColumn; + this.selectPostImage = selectPostImage; + this.columnCount = columnCount; + } + } + + // The post-images the original and transformed statements produced, compared for equality to detect the bug. + private static final class PostImages { + private final List> original; + private final List> transformed; + + PostImages(List> original, List> transformed) { + this.original = original; + this.transformed = transformed; + } + } + + // Reproduces a post-image mismatch against the reduced database. Unlike EETOracle's comparison reproducer this does + // not extend AbstractComparisonReproducer: the two sides are not independent, because the row-id stamping (UUID()) + // must run once so both observe the same rows, so both post-images are computed together. + private final class EETDMLReproducer implements Reproducer { + private final ComparisonQueries queries; + + EETDMLReproducer(ComparisonQueries queries) { + this.queries = queries; + } + + @Override + public boolean bugStillTriggers(G globalState) { + PostImages images; + try { + images = computePostImages(globalState, queries); + } catch (AssertionError | SQLException | RuntimeException e) { + // any failure re-running the comparison means this reduced database no longer shows the mismatch + return false; + } + return !images.original.equals(images.transformed); + } + + @Override + public String getBugInformation() { + StringBuilder sb = new StringBuilder(); + sb.append("-- On the database set up by the statements above, the following statements leave the database" + + " in different states:").append(System.lineSeparator()); + renderStatementLines(sb, queries); + return sb.toString(); + } + } + + // Builds the reproducer for an unexpected DBMS error, which replays the whole comparison and checks the same error + // still fires. + private UnexpectedErrorReproducer errorReproducer(ComparisonQueries queries, String expectedErrorMessage) { + UnexpectedErrorReproducer.Execution execution = globalState -> computePostImages(globalState, queries); + StringBuilder sb = new StringBuilder(); + renderStatementLines(sb, queries); + return new UnexpectedErrorReproducer<>(execution, expectedErrorMessage, sb.toString()); + } + + // Renders the failing statements as commented lines, shared by the mismatch and the unexpected-error reproducers. + private static void renderStatementLines(StringBuilder sb, ComparisonQueries queries) { + sb.append("-- original: ").append(queries.originalStatement).append(';').append(System.lineSeparator()); + sb.append("-- transformed: ").append(queries.transformedStatement).append(';').append(System.lineSeparator()); + } public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expectedErrors) { if (state == null || gen == null || expectedErrors == null) { @@ -82,6 +170,7 @@ public EETDMLOracle(G state, EETDMLGenerator gen, ExpectedErrors expect @Override public void check() throws SQLException { + reproducer = null; List tables = state.getSchema().getDatabaseTables(); if (tables.isEmpty()) { throw new IgnoreMeException(); @@ -115,29 +204,63 @@ public void check() throws SQLException { String transformedStatement = statements.transformed; generatedQueryString = originalStatement; - int columnCount = gen.postImageColumns(table).size(); + // Capture, as strings, everything needed to run and observe this comparison: the two statements plus the + // auxiliary-column setup, per-run snapshot and teardown. A reproducer replays these against a reduced database, + // where the live generator and schema objects no longer apply. + ComparisonQueries queries = new ComparisonQueries(originalStatement, transformedStatement, + gen.addRowIdColumnStatement(table), gen.stampRowIdsStatement(table), gen.beginTransactionStatement(), + gen.rollbackTransactionStatement(), gen.dropRowIdColumnStatement(table), + gen.selectPostImageStatement(table), gen.postImageColumns(table).size()); + PostImages images; + try { + images = computePostImages(state, queries); + } catch (AssertionError unexpectedError) { + reproducer = errorReproducer(queries, TestOracleUtils.getUnexpectedErrorMessage(unexpectedError)); + throw unexpectedError; + } + + reproducer = new EETDMLReproducer(queries); + if (!images.original.equals(images.transformed)) { + throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, images.original, + images.transformed)); + } + } + + /** + * Runs the whole comparison against {@code globalState}: adds and stamps the row-identifier column once (so both + * runs observe the same rows), snapshots the post-image each statement produces (each inside a rolled-back + * transaction), and drops the column. Both {@link #check()} and the reproducers call this, the former against the + * live database and the latter against a reduced one. A DBMS error the oracle tolerates aborts with + * {@link IgnoreMeException}; an oracle logic bug or unexpected error surfaces as {@link AssertionError}. + * + * @param globalState + * the state whose connection the comparison runs against + * @param queries + * the statements and auxiliary SQL to run + * + * @return the post-images the original and transformed statements produced + * + * @throws SQLException + * if a DBMS interaction fails + */ + private PostImages computePostImages(G globalState, ComparisonQueries queries) throws SQLException { // Add the auxiliary column outside the try, then guard everything after it with the finally that drops it: the // ALTER auto-commits (it is not undone by ROLLBACK), so a failure between adding and dropping would leak the // column and cause cascading duplicate-column failures - if (!new SQLQueryAdapter(gen.addRowIdColumnStatement(table), errors, true).execute(state)) { + if (!new SQLQueryAdapter(queries.addRowIdColumn, errors, true).execute(globalState)) { throw new IgnoreMeException(); } try { // Stamp identifiers once, in autocommit mode, before both runs: both then observe the same rows. - if (!new SQLQueryAdapter(gen.stampRowIdsStatement(table), errors).execute(state)) { + if (!new SQLQueryAdapter(queries.stampRowIds, errors).execute(globalState)) { throw new IgnoreMeException(); } - - List> originalImage = executeAndSnapshotPostImage(table, originalStatement, columnCount); - List> transformedImage = executeAndSnapshotPostImage(table, transformedStatement, columnCount); - - if (!originalImage.equals(transformedImage)) { - throw new AssertionError(mismatchMessage(table, originalStatement, transformedStatement, originalImage, - transformedImage)); - } + List> original = snapshotSide(globalState, queries.originalStatement, queries); + List> transformed = snapshotSide(globalState, queries.transformedStatement, queries); + return new PostImages(original, transformed); } finally { - new SQLQueryAdapter(gen.dropRowIdColumnStatement(table), errors, true).execute(state); + new SQLQueryAdapter(queries.dropRowIdColumn, errors, true).execute(globalState); } } @@ -261,12 +384,12 @@ private StatementPair generateInsertStatements(T table, E predicate, E transform * DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug or unexpected error * surfaces as {@link AssertionError}. * - * @param table - * the table being modified + * @param globalState + * the state whose connection the statement runs against * @param statement * the DML statement to execute - * @param columnCount - * the number of columns the post-image select returns (identifier plus content columns) + * @param queries + * supplies the transaction control and post-image select SQL and the post-image's column count * * @return the post-image, as one string list (identifier followed by content column values) per surviving row * @@ -274,20 +397,20 @@ private StatementPair generateInsertStatements(T table, E predicate, E transform * if a DBMS interaction other than running {@code statement} fails; an error from {@code statement} * itself instead surfaces as {@link IgnoreMeException} or {@link AssertionError} */ - private List> executeAndSnapshotPostImage(T table, String statement, int columnCount) + private List> snapshotSide(G globalState, String statement, ComparisonQueries queries) throws SQLException { - new SQLQueryAdapter(gen.beginTransactionStatement()).execute(state); + new SQLQueryAdapter(queries.beginTransaction).execute(globalState); try { // execute reports (throws AssertionError for) unexpected errors and returns false for expected ones. - boolean succeeded = new SQLQueryAdapter(statement, errors).execute(state); + boolean succeeded = new SQLQueryAdapter(statement, errors).execute(globalState); if (!succeeded) { // The statement hit an error the oracle tolerates; do not compare states (as EETOracle does for // SELECT). throw new IgnoreMeException(); } - return snapshotPostImage(gen.selectPostImageStatement(table), columnCount); + return snapshotPostImage(globalState, queries.selectPostImage, queries.columnCount); } finally { - new SQLQueryAdapter(gen.rollbackTransactionStatement()).execute(state); + new SQLQueryAdapter(queries.rollback).execute(globalState); } } @@ -296,6 +419,8 @@ private List> executeAndSnapshotPostImage(T table, String statement * {@code getString}). A DBMS error the oracle tolerates aborts with {@link IgnoreMeException}; an oracle logic bug * or unexpected error surfaces as {@link AssertionError}. * + * @param globalState + * the state whose connection the select runs against * @param selectStatement * the post-image select to read; its columns are the identifier followed by the content columns * @param columnCount @@ -307,13 +432,14 @@ private List> executeAndSnapshotPostImage(T table, String statement * if cleanup fails (errors thrown elsewhere will always be rethrown as {@link IgnoreMeException} or * {@link AssertionError}) */ - private List> snapshotPostImage(String selectStatement, int columnCount) throws SQLException { + private List> snapshotPostImage(G globalState, String selectStatement, int columnCount) + throws SQLException { List> rows = new ArrayList<>(); SQLQueryAdapter q = new SQLQueryAdapter(selectStatement, errors, true, - state.getOptions().canonicalizeSqlString()); + globalState.getOptions().canonicalizeSqlString()); SQLancerResultSet result = null; try { - result = q.executeAndGet(state); + result = q.executeAndGet(globalState); if (result == null) { throw new IgnoreMeException(); } @@ -398,4 +524,9 @@ private static String renderRow(List row) { public String getLastQueryString() { return generatedQueryString; } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } }